From 17ce7441d53e7649d771accee2e090f2a244bb2a Mon Sep 17 00:00:00 2001 From: DarkIsDude Date: Mon, 21 Sep 2026 14:39:14 +0200 Subject: [PATCH 1/6] =?UTF-8?q?=F0=9F=91=B7=20check=20prettier=20formattin?= =?UTF-8?q?g=20on=20the=20whole=20repo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prettier now owns line length, so drop the eslint max-len rule it conflicts with and the disable directives that went with it. Issue: CLDSRV-1002 --- .github/workflows/lint.yaml | 11 +--- eslint.config.mjs | 1 + lib/utilities/serverAccessLogger.js | 1 - package.json | 3 +- scripts/prettier-diff.sh | 52 ------------------- .../aws-node-sdk/test/bucket/get.js | 1 - .../test/bucket/putBucketPolicy.js | 4 +- tests/unit/api/apiUtils/permissionChecks.js | 2 - tests/unit/api/apiUtils/rateLimit/config.js | 7 --- 9 files changed, 7 insertions(+), 75 deletions(-) delete mode 100644 scripts/prettier-diff.sh diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 4476a31afb..56337747ee 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -15,18 +15,11 @@ jobs: steps: - name: Checkout uses: actions/checkout@v6 - with: - fetch-depth: 0 - uses: actions/setup-node@v6 with: node-version: '22.23.1' cache: yarn - name: install dependencies run: yarn install --frozen-lockfile --network-concurrency 1 - - name: Prettier (changed files) - shell: bash - env: - BASE_REF: ${{ github.event.pull_request.base.ref }} - run: | - MERGE_BASE=$(git merge-base HEAD "origin/${BASE_REF}") - yarn run --silent prettier:diff --check "${MERGE_BASE}..HEAD" + - name: Prettier + run: yarn run --silent prettier:check diff --git a/eslint.config.mjs b/eslint.config.mjs index d647d0f2af..043f5b7f99 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -69,6 +69,7 @@ export default [...compat.extends('@scality/scality'), { "new-parens": "off", "no-multi-spaces": "off", "quote-props": "off", + "max-len": "off", "mocha/no-exclusive-tests": "error", "no-redeclare": ["error", { "builtinGlobals": false }], "promise/prefer-await-to-then": "warn", diff --git a/lib/utilities/serverAccessLogger.js b/lib/utilities/serverAccessLogger.js index 06eae55883..23c134eba7 100644 --- a/lib/utilities/serverAccessLogger.js +++ b/lib/utilities/serverAccessLogger.js @@ -207,7 +207,6 @@ function getRemoteIPFromRequest(request) { return remoteIP; } -// eslint-disable-next-line max-len // https://github.com/awslabs/glue-extensions-for-iceberg/blob/52bdb2908216a85859fd76a45981d0326d016a2f/spark/src/main/scala/software/amazon/glue/s3a/audit/S3LogVerbs.java // https://github.com/open-io/swift/blob/ff518e9907f74b5a2565973a260f36386b5d5cbf/etc/s3-default.cfg.in#L78 // https://stackoverflow.com/questions/42707878/amazon-s3-logs-operation-definition diff --git a/package.json b/package.json index cb07ebb836..31761d26eb 100644 --- a/package.json +++ b/package.json @@ -139,7 +139,8 @@ "lint": "eslint $(git ls-files '*.js')", "lint_md": "mdlint $(git ls-files '*.md')", "prettier": "prettier", - "prettier:diff": "bash scripts/prettier-diff.sh", + "prettier:check": "prettier --check .", + "prettier:write": "prettier --write .", "mem_backend": "S3BACKEND=mem node index.js", "start": "npm-run-all --parallel start_dmd start_s3server", "start_mongo": "yarn run cloudserver", diff --git a/scripts/prettier-diff.sh b/scripts/prettier-diff.sh deleted file mode 100644 index 90db614e4a..0000000000 --- a/scripts/prettier-diff.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# Purpose: -# - Run Prettier on files returned by `git diff --name-only`. -# - Keep CI focused on changed files while remaining flexible. -# -# Usage examples: -# - scripts/prettier-diff.sh -# - scripts/prettier-diff.sh --check -# - scripts/prettier-diff.sh --format -# - scripts/prettier-diff.sh --check HEAD~1..HEAD -# - scripts/prettier-diff.sh --write --cached - -ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -cd "$ROOT_DIR" - -PRETTIER_ARGS=() -HAS_MODE=false -DIFF_ARGS=() -for arg in "$@"; do - case "$arg" in - --format) - PRETTIER_ARGS+=("--write") - HAS_MODE=true - ;; - --check|--write) - PRETTIER_ARGS+=("$arg") - HAS_MODE=true - ;; - *) - DIFF_ARGS+=("$arg") - ;; - esac -done - -if [[ "$HAS_MODE" = false ]]; then - PRETTIER_ARGS=("--check" "${PRETTIER_ARGS[@]}") -fi - -mapfile -t CHANGED < <(git diff --name-only --diff-filter=ACMRT "${DIFF_ARGS[@]}" \ - | grep -E '\.(js|cjs|mjs|ts|tsx|json|ya?ml|md)$' || true) - -if [[ ${#CHANGED[@]} -eq 0 ]]; then - echo "No supported files changed; skipping Prettier." - exit 0 -fi - -echo "Running Prettier on ${#CHANGED[@]} file(s):" -printf ' - %s\n' "${CHANGED[@]}" - -yarn run --silent prettier "${CHANGED[@]}" "${PRETTIER_ARGS[@]}" diff --git a/tests/functional/aws-node-sdk/test/bucket/get.js b/tests/functional/aws-node-sdk/test/bucket/get.js index 43f0602cca..c872a7c659 100644 --- a/tests/functional/aws-node-sdk/test/bucket/get.js +++ b/tests/functional/aws-node-sdk/test/bucket/get.js @@ -893,7 +893,6 @@ describe('GET Bucket - AWS.S3.listObjects', () => { .catch(() => {}); }); - // eslint-disable-next-line max-len it('should allow when the bucket policy supplies scality:ListBucketOptionalObjectAttributes that IAM lacks', async () => { await bucketUtil.s3.send(new PutBucketPolicyCommand({ Bucket: bucketName, diff --git a/tests/functional/aws-node-sdk/test/bucket/putBucketPolicy.js b/tests/functional/aws-node-sdk/test/bucket/putBucketPolicy.js index 9edbb25fcc..d2c12da27c 100644 --- a/tests/functional/aws-node-sdk/test/bucket/putBucketPolicy.js +++ b/tests/functional/aws-node-sdk/test/bucket/putBucketPolicy.js @@ -170,12 +170,12 @@ describe('aws-sdk test put bucket policy', () => { }); it('should allow bucket policy with pincipal arn less than 2048 characters', async () => { - const params = getPolicyParams({ key: 'Principal', value: { AWS: `arn:aws:iam::767707094035:user/${generateRandomString(150)}` } }); // eslint-disable-line max-len + const params = getPolicyParams({ key: 'Principal', value: { AWS: `arn:aws:iam::767707094035:user/${generateRandomString(150)}` } }); await s3.send(new PutBucketPolicyCommand(params)); }); it('should not allow bucket policy with pincipal arn more than 2048 characters', async () => { - const params = getPolicyParams({ key: 'Principal', value: { AWS: `arn:aws:iam::767707094035:user/${generateRandomString(2020)}` } }); // eslint-disable-line max-len + const params = getPolicyParams({ key: 'Principal', value: { AWS: `arn:aws:iam::767707094035:user/${generateRandomString(2020)}` } }); try { await s3.send(new PutBucketPolicyCommand(params)); throw new Error('Expected MalformedPolicy error'); diff --git a/tests/unit/api/apiUtils/permissionChecks.js b/tests/unit/api/apiUtils/permissionChecks.js index 03005ed541..0843682b3c 100644 --- a/tests/unit/api/apiUtils/permissionChecks.js +++ b/tests/unit/api/apiUtils/permissionChecks.js @@ -48,7 +48,6 @@ describe('authInfoHelper', () => { }); describe('checkBucketPolicy Principal logic', () => { - /* eslint-disable max-len */ const tests = [ { description: 'bucket owner with same canonicalID as requesters should return ALLOW', @@ -702,7 +701,6 @@ describe('checkBucketPolicy Principal logic', () => { expectedResult: checkBucketPolicyResult.ALLOW, }, ]; - /* eslint-enable max-len */ tests.forEach(t => { it(t.description, () => { diff --git a/tests/unit/api/apiUtils/rateLimit/config.js b/tests/unit/api/apiUtils/rateLimit/config.js index 170cde6118..bf0f84916e 100644 --- a/tests/unit/api/apiUtils/rateLimit/config.js +++ b/tests/unit/api/apiUtils/rateLimit/config.js @@ -404,7 +404,6 @@ describe('parseRateLimitConfig', () => { assert.throws( () => parseRateLimitConfig(config), - // eslint-disable-next-line max-len /rateLimiting configuration is invalid.*"bucket.defaultConfig.requestsPerSecond" must be of type object/, ); }); @@ -440,7 +439,6 @@ describe('parseRateLimitConfig', () => { assert.throws( () => parseRateLimitConfig(config), - // eslint-disable-next-line max-len /rateLimiting configuration is invalid.*"bucket.defaultConfig.requestsPerSecond.limit" must be larger than or equal to 0/, ); }); @@ -516,7 +514,6 @@ describe('parseRateLimitConfig', () => { assert.throws( () => parseRateLimitConfig(config), - // eslint-disable-next-line max-len /rateLimiting configuration is invalid.*"bucket.defaultConfig.requestsPerSecond.burstCapacity" must be larger than or equal to 0/, ); }); @@ -553,7 +550,6 @@ describe('parseRateLimitConfig', () => { assert.throws( () => parseRateLimitConfig(config), - // eslint-disable-next-line max-len /rateLimiting configuration is invalid.*"bucket.defaultConfig.requestsPerSecond.burstCapacity" must be a number/, ); }); @@ -603,7 +599,6 @@ describe('parseRateLimitConfig', () => { assert.throws( () => parseRateLimitConfig(config), - // eslint-disable-next-line max-len /rateLimiting configuration is invalid.*"bucket.defaultBurstCapacity" must be larger than or equal to 0/, ); }); @@ -1066,7 +1061,6 @@ describe('parseRateLimitConfig', () => { assert.throws( () => parseRateLimitConfig(config), - // eslint-disable-next-line max-len /rateLimiting configuration is invalid.*"account.defaultConfig.requestsPerSecond.burstCapacity" must be larger than or equal to 0/, ); }); @@ -1103,7 +1097,6 @@ describe('parseRateLimitConfig', () => { assert.throws( () => parseRateLimitConfig(config), - // eslint-disable-next-line max-len /rateLimiting configuration is invalid.*"account.defaultConfig.requestsPerSecond.burstCapacity" must be a number/, ); }); From 02f94246fe9d3f9e4484d526adc81a73e90a0540 Mon Sep 17 00:00:00 2001 From: DarkIsDude Date: Mon, 21 Sep 2026 14:44:17 +0200 Subject: [PATCH 2/6] =?UTF-8?q?=F0=9F=8E=A8=20format=20the=20whole=20codeb?= =?UTF-8?q?ase=20with=20prettier?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Generated with `yarn prettier:write`, no manual edit. Issue: CLDSRV-1002 --- .github/ISSUE_TEMPLATE.md | 12 +- .../actions/cleanup-and-coverage/action.yaml | 2 +- .github/codeql/qlpack.yml | 2 +- .github/dependabot.yml | 8 +- .github/docker/config.s3c.json | 16 +- .github/docker/docker-compose.sse.yaml | 2 +- .github/docker/docker-compose.yaml | 20 +- .github/docker/md-config.json | 2 +- .github/docker/vault-config.json | 2 +- .github/scripts/check-diff-async.mjs | 32 +- .github/scripts/cleanupOldGCPBuckets.js | 59 +- .github/scripts/count-async-functions.mjs | 40 +- .github/workflows/alerts.yaml | 2 +- .github/workflows/release.yaml | 3 +- .nycrc | 11 +- CLAUDE.md | 42 +- Healthchecks.md | 16 +- TESTING.md | 28 +- bin/metrics_server.js | 11 +- bin/search_bucket.js | 22 +- bin/secure_channel_proxy.js | 11 +- codecov.yml | 8 +- conf/authdata.json | 121 +- config.json | 53 +- constants.js | 57 +- dataserver.js | 10 +- docs/OBJECT_LOCK_TEST_PLAN.md | 42 +- docs/RELEASE.md | 64 +- eslint.config.mjs | 137 +- examples/node-md-search.js | 7 +- lib/Config.js | 1526 ++-- .../authorization/permissionChecks.js | 476 +- .../authorization/prepareRequestContexts.js | 185 +- lib/api/apiUtils/authorization/serviceUser.js | 2 +- .../authorization/tagConditionKeys.js | 103 +- lib/api/apiUtils/bucket/bucketCors.js | 173 +- lib/api/apiUtils/bucket/bucketDeletion.js | 175 +- lib/api/apiUtils/bucket/bucketEncryption.js | 51 +- lib/api/apiUtils/bucket/bucketShield.js | 38 +- lib/api/apiUtils/bucket/bucketWebsite.js | 189 +- .../bucket/checkPreferredLocations.js | 8 +- .../apiUtils/bucket/createKeyForUserBucket.js | 3 +- .../apiUtils/bucket/deleteUserBucketEntry.js | 43 +- .../bucket/getNotificationConfiguration.js | 7 +- .../bucket/getReplicationConfiguration.js | 3 +- lib/api/apiUtils/bucket/invisiblyDelete.js | 9 +- lib/api/apiUtils/bucket/parseWhere.js | 12 +- lib/api/apiUtils/bucket/updateEncryption.js | 3 +- .../bucket/validateReplicationConfig.js | 3 +- lib/api/apiUtils/bucket/validateSearch.js | 24 +- .../apiUtils/integrity/validateChecksums.js | 30 +- lib/api/apiUtils/object/applyZenkoUserMD.js | 3 +- .../apiUtils/object/checkHttpHeadersSize.js | 3 +- lib/api/apiUtils/object/checkReadLocation.js | 6 +- .../apiUtils/object/checkUserMetadataSize.js | 6 +- lib/api/apiUtils/object/coldStorage.js | 81 +- lib/api/apiUtils/object/corsResponse.js | 122 +- .../apiUtils/object/createAndStoreObject.js | 340 +- lib/api/apiUtils/object/expirationHeaders.js | 26 +- .../getReplicationBackendDataLocator.js | 25 +- lib/api/apiUtils/object/getReplicationInfo.js | 25 +- .../object/locationConstraintCheck.js | 29 +- .../apiUtils/object/locationHeaderCheck.js | 6 +- .../object/locationKeysHaveChanged.js | 28 +- .../apiUtils/object/locationStorageCheck.js | 11 +- lib/api/apiUtils/object/objectAttributes.js | 82 +- lib/api/apiUtils/object/objectLockHelpers.js | 101 +- lib/api/apiUtils/object/objectRestore.js | 124 +- lib/api/apiUtils/object/parseCopySource.js | 3 +- lib/api/apiUtils/object/partInfo.js | 9 +- lib/api/apiUtils/object/prepareStream.js | 3 +- lib/api/apiUtils/object/setPartRanges.js | 6 +- lib/api/apiUtils/object/setUpCopyLocator.js | 44 +- lib/api/apiUtils/object/sseHeaders.js | 5 +- lib/api/apiUtils/object/storeObject.js | 16 +- .../object/validateChecksumHeaders.js | 6 +- lib/api/apiUtils/object/versioning.js | 171 +- lib/api/apiUtils/object/websiteServing.js | 19 +- lib/api/apiUtils/quotas/quotaUtils.js | 311 +- lib/api/apiUtils/rateLimit/cleanup.js | 1 - lib/api/apiUtils/rateLimit/client.js | 25 +- lib/api/backbeat/listLifecycleCurrents.js | 56 +- .../listLifecycleOrphanDeleteMarkers.js | 34 +- lib/api/bucketDelete.js | 45 +- lib/api/bucketDeleteEncryption.js | 84 +- lib/api/bucketDeleteLifecycle.js | 9 +- lib/api/bucketDeleteQuota.js | 55 +- lib/api/bucketDeleteReplication.js | 9 +- lib/api/bucketDeleteTagging.js | 60 +- lib/api/bucketGet.js | 49 +- lib/api/bucketGetACL.js | 47 +- lib/api/bucketGetEncryption.js | 102 +- lib/api/bucketGetLifecycle.js | 12 +- lib/api/bucketGetLocation.js | 3 +- lib/api/bucketGetLogging.js | 66 +- lib/api/bucketGetObjectLock.js | 6 +- lib/api/bucketGetPolicy.js | 3 +- lib/api/bucketGetQuota.js | 12 +- lib/api/bucketGetRateLimit.js | 3 +- lib/api/bucketGetReplication.js | 15 +- lib/api/bucketGetTagging.js | 70 +- lib/api/bucketGetVersioning.js | 18 +- lib/api/bucketHead.js | 6 +- lib/api/bucketPutACL.js | 405 +- lib/api/bucketPutCors.js | 64 +- lib/api/bucketPutEncryption.js | 113 +- lib/api/bucketPutLifecycle.js | 89 +- lib/api/bucketPutLogging.js | 102 +- lib/api/bucketPutNotification.js | 56 +- lib/api/bucketPutObjectLock.js | 100 +- lib/api/bucketPutPolicy.js | 107 +- lib/api/bucketPutRateLimit.js | 89 +- lib/api/bucketPutReplication.js | 104 +- lib/api/bucketPutTagging.js | 72 +- lib/api/bucketPutVersioning.js | 195 +- lib/api/bucketPutWebsite.js | 69 +- lib/api/completeMultipartUpload.js | 1150 ++- lib/api/corsPreflight.js | 57 +- lib/api/initiateMultipartUpload.js | 297 +- lib/api/listMultipartUploads.js | 121 +- lib/api/listParts.js | 376 +- lib/api/metadataSearch.js | 91 +- lib/api/multipartDelete.js | 23 +- lib/api/objectCopy.js | 1009 ++- lib/api/objectDelete.js | 513 +- lib/api/objectDeleteTagging.js | 145 +- lib/api/objectGet.js | 496 +- lib/api/objectGetACL.js | 222 +- lib/api/objectGetAttributes.js | 6 +- lib/api/objectGetLegalHold.js | 3 +- lib/api/objectGetRetention.js | 129 +- lib/api/objectGetTagging.js | 119 +- lib/api/objectHead.js | 59 +- lib/api/objectPut.js | 326 +- lib/api/objectPutACL.js | 402 +- lib/api/objectPutCopyPart.js | 857 +- lib/api/objectPutLegalHold.js | 152 +- lib/api/objectPutPart.js | 713 +- lib/api/objectPutRetention.js | 183 +- lib/api/objectPutTagging.js | 154 +- lib/api/objectRestore.js | 3 +- lib/api/serviceGet.js | 43 +- lib/api/website.js | 362 +- lib/auth/in_memory/builder.js | 40 +- lib/auth/streamingV4/V4Transform.js | 54 +- .../streamingV4/constructChunkStringToSign.js | 12 +- lib/auth/vault.js | 25 +- lib/data/wrapper.js | 12 +- lib/kms/Cache.js | 5 +- lib/kms/common.js | 114 +- lib/kms/file/backend.js | 125 +- lib/kms/in_memory/backend.js | 131 +- lib/kms/utilities.js | 26 +- lib/kms/wrapper.js | 398 +- lib/management/agentClient.js | 35 +- lib/management/configuration.js | 152 +- lib/management/credentials.js | 74 +- lib/management/index.js | 115 +- lib/management/poll.js | 101 +- lib/management/push.js | 107 +- lib/metadata/acl.js | 88 +- lib/metadata/wrapper.js | 3 +- lib/nfs/utilities.js | 21 +- lib/routes/routeMetadata.js | 121 +- lib/routes/routeVeeam.js | 122 +- lib/routes/routeWorkflowEngineOperator.js | 72 +- lib/routes/utilities/pushReplicationMetric.js | 5 +- lib/routes/veeam/delete.js | 4 +- lib/routes/veeam/get.js | 9 +- lib/routes/veeam/list.js | 28 +- lib/routes/veeam/put.js | 109 +- lib/routes/veeam/schemas/capacity.js | 12 +- lib/routes/veeam/schemas/system.js | 57 +- lib/routes/veeam/utils.js | 51 +- lib/server.js | 65 +- lib/utapi/utapiReindex.js | 3 +- lib/utapi/utilities.js | 69 +- lib/utilities/aclUtils.js | 310 +- lib/utilities/collectCorsHeaders.js | 3 +- lib/utilities/collectResponseHeaders.js | 66 +- lib/utilities/healthcheckHandler.js | 45 +- lib/utilities/internalHandlers.js | 3 +- lib/utilities/legacyAWSBehavior.js | 6 +- lib/utilities/monitoringHandler.js | 114 +- lib/utilities/reportHandler.js | 432 +- lib/utilities/request.js | 14 +- lib/utilities/serverAccessLogger.js | 214 +- lib/utilities/validateQueryAndHeaders.js | 10 +- lib/utilization/scuba/wrapper.js | 64 +- managementAgent.js | 39 +- mdserver.js | 22 +- monitoring/alerts.yaml | 271 +- monitoring/dashboard.json | 6949 ++++++++-------- .../aws-node-sdk/lib/fixtures/project.js | 4 +- .../lib/json/mem_credentials.json | 32 +- .../lib/json/s3c_credentials.json | 24 +- .../aws-node-sdk/lib/utility/bucket-util.js | 115 +- .../aws-node-sdk/lib/utility/cors-util.js | 40 +- .../lib/utility/createEncryptedBucket.js | 61 +- .../lib/utility/customS3Request.js | 11 +- .../lib/utility/genMaxSizeMetaHeaders.js | 6 +- .../lib/utility/provideRawOutput.js | 10 +- .../aws-node-sdk/lib/utility/replication.js | 30 +- .../aws-node-sdk/lib/utility/tagging.js | 15 +- .../aws-node-sdk/lib/utility/test-utils.js | 2 +- .../aws-node-sdk/lib/utility/website-util.js | 255 +- .../aws-node-sdk/schema/bucket.json | 159 +- .../aws-node-sdk/schema/bucketV2.json | 181 +- .../aws-node-sdk/schema/service.json | 82 +- .../test/bucket/aclUsingPredefinedGroups.js | 429 +- .../test/bucket/bucketPolicyBypassPort.js | 217 +- .../bucketPolicyWithResourceStatements.js | 189 +- .../test/bucket/deleteBucketLifecycle.js | 28 +- .../test/bucket/deleteBucketPolicy.js | 37 +- .../test/bucket/deleteBucketQuota.js | 4 +- .../test/bucket/deleteBucketRateLimit.js | 26 +- .../test/bucket/deleteBucketReplication.js | 75 +- .../test/bucket/deleteBucketTagging.js | 74 +- .../aws-node-sdk/test/bucket/deleteCors.js | 53 +- .../aws-node-sdk/test/bucket/deleteWebsite.js | 23 +- .../aws-node-sdk/test/bucket/get.js | 912 +- .../test/bucket/getBucketEncryption.js | 19 +- .../test/bucket/getBucketLifecycle.js | 118 +- .../test/bucket/getBucketLogging.js | 38 +- .../test/bucket/getBucketNotification.js | 33 +- .../test/bucket/getBucketObjectLock.js | 28 +- .../test/bucket/getBucketPolicy.js | 47 +- .../test/bucket/getBucketQuota.js | 6 +- .../test/bucket/getBucketRateLimit.js | 17 +- .../test/bucket/getBucketReplication.js | 56 +- .../test/bucket/getBucketTagging.js | 58 +- .../aws-node-sdk/test/bucket/getCors.js | 92 +- .../aws-node-sdk/test/bucket/getLocation.js | 81 +- .../aws-node-sdk/test/bucket/getWebsite.js | 18 +- .../aws-node-sdk/test/bucket/head.js | 21 +- .../test/bucket/listingCornerCases.js | 319 +- .../aws-node-sdk/test/bucket/put.js | 397 +- .../aws-node-sdk/test/bucket/putAcl.js | 82 +- .../test/bucket/putBucketLifecycle.js | 513 +- .../test/bucket/putBucketLogging.js | 108 +- .../test/bucket/putBucketNotification.js | 57 +- .../test/bucket/putBucketObjectLock.js | 20 +- .../test/bucket/putBucketPolicy.js | 58 +- .../test/bucket/putBucketRateLimit.js | 113 +- .../test/bucket/putBucketReplication.js | 345 +- .../test/bucket/putBucketTagging.js | 178 +- .../aws-node-sdk/test/bucket/putCors.js | 88 +- .../aws-node-sdk/test/bucket/putWebsite.js | 170 +- .../aws-node-sdk/test/bucket/skipScan.js | 31 +- .../test/bucket/testBucketStress.js | 6 +- .../test/bucket/testBucketVersioning.js | 65 +- .../test/bucket/updateBucketQuota.js | 22 +- .../test/legacy/authV2QueryTests.js | 28 +- .../test/legacy/authV4QueryTests.js | 28 +- .../aws-node-sdk/test/legacy/tests.js | 73 +- .../aws-node-sdk/test/mdSearch/basicSearch.js | 125 +- .../test/mdSearch/utils/helpers.js | 84 +- .../test/mdSearch/versionEnabledSearch.js | 53 +- .../multipleBackend/acl/aclAwsVersioning.js | 229 +- .../test/multipleBackend/delete/delete.js | 122 +- .../delete/deleteAwsVersioning.js | 1344 +-- .../multipleBackend/delete/deleteAzure.js | 418 +- .../test/multipleBackend/delete/deleteGcp.js | 154 +- .../test/multipleBackend/get/get.js | 428 +- .../multipleBackend/get/getAwsVersioning.js | 923 +- .../test/multipleBackend/get/getAzure.js | 238 +- .../test/multipleBackend/get/getGcp.js | 168 +- .../multipleBackend/initMPU/initMPUAzure.js | 106 +- .../multipleBackend/initMPU/initMPUGcp.js | 126 +- .../listParts/azureListParts.js | 176 +- .../multipleBackend/listParts/listPartsGcp.js | 175 +- .../multipleBackend/mpuAbort/abortMPUGcp.js | 340 +- .../multipleBackend/mpuAbort/azureAbortMPU.js | 342 +- .../mpuComplete/azureCompleteMPU.js | 266 +- .../mpuComplete/completeMPUGcp.js | 232 +- .../mpuComplete/mpuAwsVersioning.js | 299 +- .../multipleBackend/mpuParts/azurePutPart.js | 752 +- .../multipleBackend/mpuParts/putPartGcp.js | 640 +- .../objectCopy/azureObjectCopy.js | 979 ++- .../multipleBackend/objectCopy/objectCopy.js | 693 +- .../objectCopy/objectCopyAwsVersioning.js | 682 +- .../objectPutCopyPartAzure.js | 1302 +-- .../objectPutCopyPart/objectPutCopyPartGcp.js | 1168 +-- .../objectTagging/objectTagging.js | 389 +- .../taggingAwsVersioning-delete.js | 463 +- .../taggingAwsVersioning-putget.js | 765 +- .../test/multipleBackend/put/put.js | 808 +- .../test/multipleBackend/put/putAzure.js | 430 +- .../test/multipleBackend/put/putGcp.js | 263 +- .../test/multipleBackend/unknownEndpoint.js | 36 +- .../test/multipleBackend/utils.js | 247 +- .../aws-node-sdk/test/object/100-continue.js | 27 +- .../aws-node-sdk/test/object/abortMPU.js | 1046 ++- .../aws-node-sdk/test/object/bigMpu.js | 133 +- .../aws-node-sdk/test/object/completeMPU.js | 398 +- .../aws-node-sdk/test/object/compluteMpu.js | 17 +- .../aws-node-sdk/test/object/copyPart.js | 1584 ++-- .../test/object/corsErrorHeaders.js | 133 +- .../aws-node-sdk/test/object/corsHeaders.js | 300 +- .../aws-node-sdk/test/object/corsPreflight.js | 1058 ++- .../aws-node-sdk/test/object/deleteMpu.js | 81 +- .../test/object/deleteObjTagging.js | 237 +- .../aws-node-sdk/test/object/deleteObject.js | 511 +- .../test/object/encryptionHeaders.js | 216 +- .../aws-node-sdk/test/object/get.js | 1528 ++-- .../test/object/getMPU_compatibleHeaders.js | 220 +- .../aws-node-sdk/test/object/getObjTagging.js | 252 +- .../test/object/getObjectLegalHold.js | 213 +- .../aws-node-sdk/test/object/getPartSize.js | 124 +- .../aws-node-sdk/test/object/getRange.js | 31 +- .../aws-node-sdk/test/object/getRetention.js | 133 +- .../aws-node-sdk/test/object/initiateMPU.js | 244 +- .../aws-node-sdk/test/object/listParts.js | 171 +- .../aws-node-sdk/test/object/mpu.js | 82 +- .../aws-node-sdk/test/object/mpuOrder.js | 156 +- .../test/object/multiObjectDelete.js | 508 +- .../aws-node-sdk/test/object/objectCopy.js | 2585 +++--- .../test/object/objectGetAttributes.js | 530 +- .../aws-node-sdk/test/object/objectHead.js | 719 +- .../object/objectHead_compatibleHeaders.js | 122 +- .../test/object/objectHead_replication.js | 74 +- .../test/object/objectOverwrite.js | 177 +- .../aws-node-sdk/test/object/put.js | 703 +- .../aws-node-sdk/test/object/putObjAcl.js | 53 +- .../aws-node-sdk/test/object/putObjTagging.js | 318 +- .../test/object/putObjectLegalHold.js | 275 +- .../aws-node-sdk/test/object/putPart.js | 62 +- .../aws-node-sdk/test/object/putRetention.js | 155 +- .../aws-node-sdk/test/object/putVersion.js | 1572 ++-- .../aws-node-sdk/test/object/rangeTest.js | 299 +- .../test/object/websiteFiles/error.html | 14 +- .../test/object/websiteFiles/index.html | 17 +- .../test/object/websiteFiles/redirect.html | 12 +- .../aws-node-sdk/test/object/websiteGet.js | 1135 +-- .../test/object/websiteGetWithACL.js | 99 +- .../aws-node-sdk/test/object/websiteHead.js | 655 +- .../test/object/websiteHeadWithACL.js | 83 +- .../test/object/websiteRuleMixing.js | 677 +- .../aws-node-sdk/test/quota/tooling.js | 4 +- .../aws-node-sdk/test/rateLimit/client.js | 2 +- .../aws-node-sdk/test/service/get.js | 211 +- .../aws-node-sdk/test/support/awsConfig.js | 42 +- .../test/support/objectConfigs.js | 10 +- .../aws-node-sdk/test/support/withV4.js | 12 +- .../aws-node-sdk/test/utils/init.js | 45 +- .../test/versioning/bucketDelete.js | 88 +- .../versioning/legacyNullVersionCompat.js | 275 +- .../versioning/listObjectMasterVersions.js | 76 +- .../test/versioning/listObjectVersions.js | 136 +- .../test/versioning/multiObjectDelete.js | 297 +- .../aws-node-sdk/test/versioning/objectACL.js | 408 +- .../test/versioning/objectCopy.js | 1025 ++- .../test/versioning/objectDelete.js | 536 +- .../test/versioning/objectDeleteTagging.js | 357 +- .../aws-node-sdk/test/versioning/objectGet.js | 227 +- .../test/versioning/objectGetAttributes.js | 123 +- .../test/versioning/objectGetTagging.js | 399 +- .../test/versioning/objectHead.js | 511 +- .../aws-node-sdk/test/versioning/objectPut.js | 424 +- .../test/versioning/objectPutCopyPart.js | 724 +- .../test/versioning/objectPutTagging.js | 559 +- .../test/versioning/replicationBucket.js | 88 +- .../test/versioning/versioningGeneral1.js | 193 +- .../test/versioning/versioningGeneral2.js | 368 +- tests/functional/backbeat/bucketIndexing.js | 233 +- .../backbeat/excludedDataStoreName.js | 550 +- tests/functional/backbeat/listDeleteMarker.js | 357 +- .../backbeat/listLifecycleCurrents.js | 1296 +-- .../listLifecycleOrphanDeleteMarkers.js | 947 ++- tests/functional/backbeat/listNullVersion.js | 351 +- tests/functional/backbeat/utils.js | 4 +- tests/functional/healthchecks/package.json | 29 +- .../healthchecks/test/checkRoutes.js | 9 +- .../functional/kmip/serverside_encryption.js | 101 +- .../functional/metadata/MixedVersionFormat.js | 319 +- .../metadata/MongoClientInterface.js | 378 +- tests/functional/raw-node/package.json | 42 +- tests/functional/raw-node/test/GCP/README.MD | 4 +- .../raw-node/test/GCP/bucket/bucket.js | 75 +- .../raw-node/test/GCP/bucket/versioning.js | 33 +- .../raw-node/test/GCP/object/completeMpu.js | 157 +- .../raw-node/test/GCP/object/deleteMpu.js | 207 +- .../raw-node/test/GCP/object/initiateMpu.js | 83 +- .../raw-node/test/GCP/object/object.js | 454 +- .../raw-node/test/GCP/object/tagging.js | 449 +- .../raw-node/test/GCP/object/upload.js | 58 +- .../raw-node/test/badChunkSignatureV4.js | 99 +- tests/functional/raw-node/test/headObject.js | 59 +- tests/functional/raw-node/test/lifecycle.js | 190 +- .../raw-node/test/malformedDateHeader.js | 16 +- .../raw-node/test/routes/routeMetadata.js | 146 +- .../raw-node/test/trailingChecksums.js | 102 +- .../raw-node/test/unsignedChecksumHeaders.js | 86 +- .../raw-node/test/unsupportedChecksums.js | 42 +- .../raw-node/test/unsupportedQuries.js | 12 +- .../raw-node/utils/HttpRequestAuthV4.js | 102 +- .../functional/raw-node/utils/MetadataMock.js | 381 +- tests/functional/raw-node/utils/gcpUtils.js | 176 +- .../functional/raw-node/utils/makeRequest.js | 45 +- tests/functional/report/master.json | 2 +- tests/functional/report/monitoring.js | 55 +- tests/functional/s3cmd/tests.js | 274 +- tests/functional/s3curl/tests.js | 1160 ++- .../functional/sse-kms-migration/arnPrefix.js | 857 +- .../sse-kms-migration/beforeMigration.js | 826 +- tests/functional/sse-kms-migration/cleanup.js | 20 +- .../sse-kms-migration/configs/aws.json | 3 +- .../sse-kms-migration/configs/base.json | 49 +- .../configs/kmip-cluster.json | 2 +- tests/functional/sse-kms-migration/helpers.js | 71 +- tests/functional/sse-kms-migration/load.js | 69 +- .../functional/sse-kms-migration/migration.js | 585 +- .../functional/sse-kms-migration/scenarios.js | 33 +- tests/functional/utilities/reportHandler.js | 183 +- .../locationConfig/locationConfigLegacy.json | 1 - .../backendHealthcheckResponse.js | 179 +- tests/multipleBackend/multipartUpload.js | 751 +- tests/multipleBackend/objectCopy.js | 80 +- tests/multipleBackend/objectPut.js | 68 +- tests/multipleBackend/objectPutCopyPart.js | 276 +- tests/multipleBackend/objectPutPart.js | 264 +- tests/multipleBackend/routes/routeBackbeat.js | 7396 ++++++++++------- tests/sur/quota.js | 1563 ++-- tests/sur/routeVeeam.js | 885 +- tests/unit/Config.js | 259 +- .../api/apiUtils/authorization/aclChecks.js | 22 +- .../authorization/prepareRequestContexts.js | 479 +- tests/unit/api/apiUtils/coldStorage.js | 174 +- tests/unit/api/apiUtils/expirationHeaders.js | 34 +- .../apiUtils/getNotificationConfiguration.js | 33 +- tests/unit/api/apiUtils/getReplicationInfo.js | 315 +- .../apiUtils/integrity/validateChecksums.js | 75 +- .../api/apiUtils/locationKeysHaveChanged.js | 3 +- .../api/apiUtils/object/objectAttributes.js | 9 +- tests/unit/api/apiUtils/objectLockHelpers.js | 211 +- tests/unit/api/apiUtils/permissionChecks.js | 353 +- tests/unit/api/apiUtils/quotas/quotaUtils.js | 783 +- tests/unit/api/apiUtils/rateLimit/cache.js | 11 +- tests/unit/api/apiUtils/rateLimit/cleanup.js | 13 +- tests/unit/api/apiUtils/tagConditionKeys.js | 43 +- .../api/apiUtils/validateChecksumHeaders.js | 1 - tests/unit/api/apiUtils/versioning.js | 546 +- tests/unit/api/bucketACLauth.js | 235 +- tests/unit/api/bucketDelete.js | 152 +- tests/unit/api/bucketDeleteCors.js | 16 +- tests/unit/api/bucketDeleteEncryption.js | 3 +- tests/unit/api/bucketDeleteLifecycle.js | 45 +- tests/unit/api/bucketDeletePolicy.js | 32 +- tests/unit/api/bucketDeleteTagging.js | 56 +- tests/unit/api/bucketDeleteWebsite.js | 20 +- tests/unit/api/bucketGet.js | 894 +- tests/unit/api/bucketGetACL.js | 517 +- tests/unit/api/bucketGetCors.js | 25 +- tests/unit/api/bucketGetLifecycle.js | 17 +- tests/unit/api/bucketGetLocation.js | 46 +- tests/unit/api/bucketGetLogging.js | 12 +- tests/unit/api/bucketGetNotification.js | 18 +- tests/unit/api/bucketGetObjectLock.js | 30 +- tests/unit/api/bucketGetPolicy.js | 8 +- tests/unit/api/bucketGetRateLimit.js | 3 +- tests/unit/api/bucketGetReplication.js | 15 +- tests/unit/api/bucketGetTagging.js | 25 +- tests/unit/api/bucketGetWebsite.js | 20 +- tests/unit/api/bucketHead.js | 3 +- tests/unit/api/bucketPolicyAuth.js | 342 +- tests/unit/api/bucketPutACL.js | 615 +- tests/unit/api/bucketPutCors.js | 60 +- tests/unit/api/bucketPutEncryption.js | 151 +- tests/unit/api/bucketPutLifecycle.js | 19 +- tests/unit/api/bucketPutLogging.js | 21 +- tests/unit/api/bucketPutNotification.js | 31 +- tests/unit/api/bucketPutObjectLock.js | 21 +- tests/unit/api/bucketPutPolicy.js | 31 +- tests/unit/api/bucketPutReplication.js | 194 +- tests/unit/api/bucketPutTagging.js | 12 +- tests/unit/api/bucketPutVersioning.js | 161 +- tests/unit/api/bucketPutWebsite.js | 203 +- tests/unit/api/corsErrorHeaders.js | 387 +- tests/unit/api/createAndStoreObject.js | 724 +- tests/unit/api/deleteMarker.js | 109 +- tests/unit/api/deletedFlagBucket.js | 587 +- tests/unit/api/listMultipartUploads.js | 226 +- tests/unit/api/listParts.js | 267 +- tests/unit/api/multipartDelete.js | 164 +- tests/unit/api/multipartUpload.js | 4954 +++++------ tests/unit/api/objectACLauth.js | 280 +- tests/unit/api/objectCopy.js | 697 +- tests/unit/api/objectCopyPart.js | 72 +- tests/unit/api/objectDelete.js | 462 +- tests/unit/api/objectDeleteTagging.js | 57 +- tests/unit/api/objectGet.js | 499 +- tests/unit/api/objectGetACL.js | 652 +- tests/unit/api/objectGetAttributes.js | 33 +- tests/unit/api/objectGetLegalHold.js | 76 +- tests/unit/api/objectGetRetention.js | 33 +- tests/unit/api/objectGetTagging.js | 35 +- tests/unit/api/objectHead.js | 500 +- tests/unit/api/objectPut.js | 1704 ++-- tests/unit/api/objectPutACL.js | 620 +- tests/unit/api/objectPutLegalHold.js | 43 +- tests/unit/api/objectPutRetention.js | 127 +- tests/unit/api/objectPutTagging.js | 97 +- tests/unit/api/objectReplicationMD.js | 926 ++- tests/unit/api/objectRestore.js | 129 +- tests/unit/api/parseLikeExpression.js | 26 +- tests/unit/api/serviceGet.js | 54 +- tests/unit/api/transientBucket.js | 436 +- .../unit/api/utils/metadataMockColdStorage.js | 56 +- tests/unit/auth/TrailingChecksumTransform.js | 15 +- tests/unit/auth/V4Transform.js | 43 +- tests/unit/auth/in_memory/backend.js | 21 +- tests/unit/auth/permissionChecks.js | 277 +- tests/unit/bucket/bucket_mem_api.js | 382 +- tests/unit/encryption/checkHealth.js | 22 +- tests/unit/encryption/healthCheckCache.js | 16 +- tests/unit/encryption/kms.js | 205 +- .../githubScripts/asyncMigrationScripts.js | 10 +- tests/unit/healthchecks/clientCheck.js | 265 +- tests/unit/helpers.js | 125 +- tests/unit/internal/routeVeeam.js | 39 +- tests/unit/internal/veeam/schemas/system.js | 5 +- tests/unit/management/agent.js | 104 +- tests/unit/management/configuration.js | 111 +- tests/unit/management/secureChannel.js | 22 +- tests/unit/management/testChannelMessageV0.js | 6 +- .../VersioningBackendClient.js | 87 +- .../getReplicationBackendDataLocator.js | 51 +- .../locationConstraintCheck.js | 70 +- .../multipleBackend/locationHeaderCheck.js | 21 +- tests/unit/policies.js | 53 +- tests/unit/quotas/scuba/wrapper.js | 4 +- tests/unit/routes/veeam-routes.js | 96 +- tests/unit/routes/veeam-utils.js | 10 +- tests/unit/server.js | 73 +- .../testConfigs/allOptsConfig/config.json | 61 +- .../unit/testConfigs/bucketNotifConfigTest.js | 83 +- tests/unit/testConfigs/configTest.js | 307 +- tests/unit/testConfigs/locConstraintAssert.js | 405 +- tests/unit/testConfigs/parseKmsAWS.js | 2 +- .../unit/testConfigs/parseRedisConfig.spec.js | 6 +- tests/unit/testConfigs/parseSproxydConfig.js | 9 +- tests/unit/testConfigs/requestsConfigTest.js | 205 +- tests/unit/utils/aclUtils.js | 39 +- tests/unit/utils/bucketEncryption.js | 4 +- tests/unit/utils/checkReadLocation.js | 12 +- tests/unit/utils/collectResponseHeaders.js | 23 +- tests/unit/utils/gcpMpuHelpers.js | 7 +- tests/unit/utils/gcpTaggingHelpers.js | 27 +- tests/unit/utils/lifecycleHelpers.js | 6 +- tests/unit/utils/monitoring.js | 49 +- tests/unit/utils/mpuUtils.js | 95 +- tests/unit/utils/multipleBackendGateway.js | 57 +- tests/unit/utils/pushReplicationMetric.js | 93 +- tests/unit/utils/request.js | 214 +- tests/unit/utils/responseStreamData.js | 69 +- tests/unit/utils/serverAccessLogger.js | 92 +- tests/unit/utils/setPartRanges.js | 97 +- tests/unit/utils/validateSearch.js | 96 +- tests/utapi/awsNodeSdk.js | 601 +- tests/utapi/utilities.js | 1048 +-- tests/utilities/bucketTagging-util.js | 14 +- tests/utilities/mock/Scuba.js | 5 +- tests/utilities/objectLock-util.js | 32 +- yamllint.yml | 1 - 564 files changed, 68597 insertions(+), 59068 deletions(-) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 6553e014e9..636b0e4395 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -9,7 +9,7 @@ the **[Zenko Forum](http://forum.zenko.io/)**. > Questions opened as GitHub issues will systematically be closed, and moved to > the [Zenko Forum](http://forum.zenko.io/). --------------------------------------------------------------------------------- +--- ## Avoiding duplicates @@ -21,7 +21,7 @@ any duplicates already open: - if there is a duplicate, please do not open your issue, and add a comment to the existing issue instead. --------------------------------------------------------------------------------- +--- ## Bug report information @@ -52,7 +52,7 @@ Describe the results you expected - distribution/OS, - optional: anything else you deem helpful to us. --------------------------------------------------------------------------------- +--- ## Feature Request @@ -78,10 +78,10 @@ Please provide use cases for changing the current behavior ### Additional information - Is this request for your company? Y/N - - If Y: Company name: - - Are you using any Scality Enterprise Edition products (RING, Zenko EE)? Y/N + - If Y: Company name: + - Are you using any Scality Enterprise Edition products (RING, Zenko EE)? Y/N - Are you willing to contribute this feature yourself? - Position/Title: - How did you hear about us? --------------------------------------------------------------------------------- +--- diff --git a/.github/actions/cleanup-and-coverage/action.yaml b/.github/actions/cleanup-and-coverage/action.yaml index f783b9518d..562d58e1d3 100644 --- a/.github/actions/cleanup-and-coverage/action.yaml +++ b/.github/actions/cleanup-and-coverage/action.yaml @@ -34,7 +34,7 @@ runs: env: INPUT_PROFILES: ${{ inputs.profiles }} working-directory: .github/docker - + - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v5 with: diff --git a/.github/codeql/qlpack.yml b/.github/codeql/qlpack.yml index 8a73ff8cbc..6d7a58f725 100644 --- a/.github/codeql/qlpack.yml +++ b/.github/codeql/qlpack.yml @@ -1,4 +1,4 @@ name: scality/cloudserver-async-migration version: 0.0.1 dependencies: - codeql/javascript-all: "*" + codeql/javascript-all: '*' diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 656e5f645f..7c4b762e2a 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -6,7 +6,7 @@ updates: interval: weekly open-pull-requests-limit: 5 ignore: - - dependency-name: "scality/vault" - update-types: ["version-update:semver-major"] - - dependency-name: "scality/metadata" - update-types: ["version-update:semver-major"] + - dependency-name: 'scality/vault' + update-types: ['version-update:semver-major'] + - dependency-name: 'scality/metadata' + update-types: ['version-update:semver-major'] diff --git a/.github/docker/config.s3c.json b/.github/docker/config.s3c.json index f0f0744b8d..f2795a325a 100644 --- a/.github/docker/config.s3c.json +++ b/.github/docker/config.s3c.json @@ -35,25 +35,17 @@ "dumpLevel": "error" }, "replicationEndpoints": [ - {"site": "zenko", "servers": ["127.0.0.1:9080"], "echo": false, "default": true}, - {"site": "us-east-2", "servers": ["127.0.0.1:9080"]} + { "site": "zenko", "servers": ["127.0.0.1:9080"], "echo": false, "default": true }, + { "site": "us-east-2", "servers": ["127.0.0.1:9080"] } ], "requests": { "extractClientIPFromHeader": "x-forwarded-for", "extractProtocolFromHeader": "x-forwarded-proto", - "trustedProxyCIDRs": [ - "127.0.0.1/32", - "::ffff:127.0.0.1/128", - "127.0.0.1" - ], + "trustedProxyCIDRs": ["127.0.0.1/32", "::ffff:127.0.0.1/128", "127.0.0.1"], "viaProxy": true }, "multiObjectDeleteEnableOptimizations": false, - "supportedLifecycleRules": [ - "Expiration", - "NoncurrentVersionExpiration", - "AbortIncompleteMultipartUpload" - ], + "supportedLifecycleRules": ["Expiration", "NoncurrentVersionExpiration", "AbortIncompleteMultipartUpload"], "bucketNotificationDestinations": [ { "resource": "target1", diff --git a/.github/docker/docker-compose.sse.yaml b/.github/docker/docker-compose.sse.yaml index f902ba81d8..4b50563154 100644 --- a/.github/docker/docker-compose.sse.yaml +++ b/.github/docker/docker-compose.sse.yaml @@ -5,7 +5,7 @@ services: # root because S3C images needs ownership permission on files and mounted paths user: root command: sh -c "chmod 400 tests/utils/keyfile && yarn start > /artifacts/vault.log 2> /artifacts/vault-stderr.log" - network_mode: "host" + network_mode: 'host' volumes: - /tmp/artifacts/${JOB_NAME}:/artifacts - ./vault-config.json:/conf/config.json:ro diff --git a/.github/docker/docker-compose.yaml b/.github/docker/docker-compose.yaml index 370ced54f6..d61e499020 100644 --- a/.github/docker/docker-compose.yaml +++ b/.github/docker/docker-compose.yaml @@ -1,7 +1,7 @@ services: cloudserver: image: ${CLOUDSERVER_IMAGE} - network_mode: "host" + network_mode: 'host' volumes: - /tmp/ssl:/ssl - /tmp/ssl-kmip:/tmp/ssl-kmip @@ -56,13 +56,13 @@ services: depends_on: - redis extra_hosts: - - "bucketwebsitetester.s3-website-us-east-1.amazonaws.com:127.0.0.1" - - "pykmip.local:127.0.0.1" + - 'bucketwebsitetester.s3-website-us-east-1.amazonaws.com:127.0.0.1' + - 'pykmip.local:127.0.0.1' redis: image: redis:alpine - network_mode: "host" + network_mode: 'host' squid: - network_mode: "host" + network_mode: 'host' profiles: ['ci-proxy'] image: scality/ci-squid command: >- @@ -76,7 +76,7 @@ services: volumes: - /tmp/ssl:/ssl pykmip: - network_mode: "host" + network_mode: 'host' profiles: ['pykmip'] image: ${PYKMIP_IMAGE:-ghcr.io/scality/cloudserver/pykmip} volumes: @@ -86,17 +86,17 @@ services: - ../pykmip/policy.json:/etc/pykmip/policies/policy.json - ../pykmip/server.conf:/etc/pykmip/server.conf localkms: - network_mode: "host" + network_mode: 'host' profiles: ['localkms'] image: ${KMS_IMAGE:-nsmithuk/local-kms:3.11.7} mongo: - network_mode: "host" + network_mode: 'host' profiles: ['mongo'] image: ${MONGODB_IMAGE} volumes: - /tmp/artifacts/${JOB_NAME}:/logs sproxyd: - network_mode: "host" + network_mode: 'host' profiles: ['sproxyd'] image: sproxyd-standalone build: ./sproxyd @@ -109,7 +109,7 @@ services: profiles: ['vault'] user: root command: sh -c "chmod 400 tests/utils/keyfile && yarn start > /artifacts/vault.log 2> /artifacts/vault-stderr.log" - network_mode: "host" + network_mode: 'host' volumes: - /tmp/artifacts/${JOB_NAME}:/artifacts - ./vault-config.json:/conf/config.json:ro diff --git a/.github/docker/md-config.json b/.github/docker/md-config.json index 405c350db5..05afaf5da4 100644 --- a/.github/docker/md-config.json +++ b/.github/docker/md-config.json @@ -11,7 +11,7 @@ "logLevel": "info", "env": { "METADATA_NEW_BUCKETS_VFORMAT": "v0", - "S3_VERSION_ID_ENCODING_TYPE":"hex" + "S3_VERSION_ID_ENCODING_TYPE": "hex" }, "migration": { "deploy": false, diff --git a/.github/docker/vault-config.json b/.github/docker/vault-config.json index 6f558cf5c4..ef00a63126 100644 --- a/.github/docker/vault-config.json +++ b/.github/docker/vault-config.json @@ -68,7 +68,7 @@ "kmsAWS": { "noAwsArn": true, "providerName": "local", - "region": "us-east-1", + "region": "us-east-1", "endpoint": "http://0:8080", "ak": "456", "sk": "123" diff --git a/.github/scripts/check-diff-async.mjs b/.github/scripts/check-diff-async.mjs index 5d4564d193..d2c7f7302f 100644 --- a/.github/scripts/check-diff-async.mjs +++ b/.github/scripts/check-diff-async.mjs @@ -11,17 +11,10 @@ import { Project, SyntaxKind } from 'ts-morph'; const CALLBACK_PARAM_PATTERN = /^(cb|callback|next|done)$/i; function getChangedJsFiles() { - const base = process.env.GITHUB_BASE_REF - ? `origin/${process.env.GITHUB_BASE_REF}` - : 'HEAD'; - const output = execFileSync('git', [ - 'diff', - '--name-only', - '--diff-filter=ACMR', - base, - '--', - '**/*.js', - ], { encoding: 'utf8' }).trim(); + const base = process.env.GITHUB_BASE_REF ? `origin/${process.env.GITHUB_BASE_REF}` : 'HEAD'; + const output = execFileSync('git', ['diff', '--name-only', '--diff-filter=ACMR', base, '--', '**/*.js'], { + encoding: 'utf8', + }).trim(); return output ? output.split('\n').filter(f => f.endsWith('.js')) : []; } @@ -30,9 +23,7 @@ function getChangedJsFiles() { * Get added line numbers for a file in the current diff. */ function getAddedLineNumbers(filePath) { - const base = process.env.GITHUB_BASE_REF - ? `origin/${process.env.GITHUB_BASE_REF}` - : 'HEAD'; + const base = process.env.GITHUB_BASE_REF ? `origin/${process.env.GITHUB_BASE_REF}` : 'HEAD'; const diff = execFileSync('git', ['diff', base, '--', filePath], { encoding: 'utf8' }); const addedLines = new Set(); let currentLine = 0; @@ -69,14 +60,11 @@ const project = new Project({ skipAddingFilesFromTsConfig: true, }); -const filesToCheck = changedFiles.filter(f => - !f.startsWith('tests/') && - !f.startsWith('node_modules/') && - ( - f.startsWith('lib/') || - f.startsWith('bin/') || - !f.includes('/') - ) +const filesToCheck = changedFiles.filter( + f => + !f.startsWith('tests/') && + !f.startsWith('node_modules/') && + (f.startsWith('lib/') || f.startsWith('bin/') || !f.includes('/')), ); if (filesToCheck.length === 0) { console.log('No source JS files in diff (tests and node_modules excluded).'); diff --git a/.github/scripts/cleanupOldGCPBuckets.js b/.github/scripts/cleanupOldGCPBuckets.js index 146fc7bf2a..767a79524f 100644 --- a/.github/scripts/cleanupOldGCPBuckets.js +++ b/.github/scripts/cleanupOldGCPBuckets.js @@ -21,8 +21,7 @@ function buildClient() { if (!accessKeyId || !secretAccessKey) { console.error( - 'Missing required environment variables: ' + - 'AWS_GCP_BACKEND_ACCESS_KEY and AWS_GCP_BACKEND_SECRET_KEY' + 'Missing required environment variables: ' + 'AWS_GCP_BACKEND_ACCESS_KEY and AWS_GCP_BACKEND_SECRET_KEY', ); process.exit(1); } @@ -56,21 +55,23 @@ async function abortMultipartUploads(client, bucketName) { let keyMarker; do { - const res = await client.send(new ListMultipartUploadsCommand({ - Bucket: bucketName, - UploadIdMarker: uploadIdMarker, - KeyMarker: keyMarker, - })); + const res = await client.send( + new ListMultipartUploadsCommand({ + Bucket: bucketName, + UploadIdMarker: uploadIdMarker, + KeyMarker: keyMarker, + }), + ); for (const upload of res.Uploads || []) { - console.log( - ` Aborting MPU: ${upload.Key} (${upload.UploadId})` + console.log(` Aborting MPU: ${upload.Key} (${upload.UploadId})`); + await client.send( + new AbortMultipartUploadCommand({ + Bucket: bucketName, + Key: upload.Key, + UploadId: upload.UploadId, + }), ); - await client.send(new AbortMultipartUploadCommand({ - Bucket: bucketName, - Key: upload.Key, - UploadId: upload.UploadId, - })); } uploadIdMarker = res.NextUploadIdMarker; @@ -82,19 +83,23 @@ async function deleteAllObjects(client, bucketName) { let continuationToken; do { - const res = await client.send(new ListObjectsV2Command({ - Bucket: bucketName, - ContinuationToken: continuationToken, - })); + const res = await client.send( + new ListObjectsV2Command({ + Bucket: bucketName, + ContinuationToken: continuationToken, + }), + ); const objects = res.Contents || []; if (objects.length > 0) { console.log(` Deleting ${objects.length} object(s)...`); for (const obj of objects) { - await client.send(new DeleteObjectCommand({ - Bucket: bucketName, - Key: obj.Key, - })); + await client.send( + new DeleteObjectCommand({ + Bucket: bucketName, + Key: obj.Key, + }), + ); } } @@ -121,9 +126,8 @@ async function main() { const { Buckets = [] } = await client.send(new ListBucketsCommand({})); const now = Date.now(); - const stale = Buckets.filter(b => - b.Name.startsWith(BUCKET_PREFIX) && - now - new Date(b.CreationDate).getTime() > ONE_WEEK_MS + const stale = Buckets.filter( + b => b.Name.startsWith(BUCKET_PREFIX) && now - new Date(b.CreationDate).getTime() > ONE_WEEK_MS, ); if (stale.length === 0) { @@ -131,10 +135,7 @@ async function main() { return; } - console.log( - `Found ${stale.length} stale GCP CI bucket(s) to clean up: ${ - stale.map(b => b.Name).join(', ')}` - ); + console.log(`Found ${stale.length} stale GCP CI bucket(s) to clean up: ${stale.map(b => b.Name).join(', ')}`); for (const bucket of stale) { await cleanupBucket(client, bucket.Name); diff --git a/.github/scripts/count-async-functions.mjs b/.github/scripts/count-async-functions.mjs index 48fd4b65ea..895fb7751c 100644 --- a/.github/scripts/count-async-functions.mjs +++ b/.github/scripts/count-async-functions.mjs @@ -67,13 +67,12 @@ for (const sourceFile of project.getSourceFiles()) { } } -const asyncFunctionPercent = totalFunctions > 0 - ? ((asyncFunctions / totalFunctions) * 100).toFixed(1) - : '0.0'; +const asyncFunctionPercent = totalFunctions > 0 ? ((asyncFunctions / totalFunctions) * 100).toFixed(1) : '0.0'; -const migrationPercent = (asyncFunctions + callbackFunctions) > 0 - ? ((asyncFunctions / (asyncFunctions + callbackFunctions)) * 100).toFixed(1) - : '0.0'; +const migrationPercent = + asyncFunctions + callbackFunctions > 0 + ? ((asyncFunctions / (asyncFunctions + callbackFunctions)) * 100).toFixed(1) + : '0.0'; console.log('=== Async/Await Migration Progress ==='); console.log(`Total functions: ${totalFunctions}`); @@ -84,18 +83,21 @@ console.log(''); console.log(`Migration (trend): ${asyncFunctions}/${asyncFunctions + callbackFunctions} (${migrationPercent}%)`); if (process.env.GITHUB_STEP_SUMMARY) { - appendFileSync(process.env.GITHUB_STEP_SUMMARY, [ - '## Async/Await Migration Progress', - '', - `| Metric | Count |`, - `|--------|-------|`, - `| Total functions | ${totalFunctions} |`, - `| Async functions | ${asyncFunctions} (${asyncFunctionPercent}%) |`, - `| Callback-style functions | ${callbackFunctions} |`, - `| Remaining \`.then()\` chains | ${thenChains} |`, - `| Migration trend (async / (async + callback)) | ${asyncFunctions}/${asyncFunctions + callbackFunctions} (${migrationPercent}%) |`, - '', - ].join('\n')); + appendFileSync( + process.env.GITHUB_STEP_SUMMARY, + [ + '## Async/Await Migration Progress', + '', + `| Metric | Count |`, + `|--------|-------|`, + `| Total functions | ${totalFunctions} |`, + `| Async functions | ${asyncFunctions} (${asyncFunctionPercent}%) |`, + `| Callback-style functions | ${callbackFunctions} |`, + `| Remaining \`.then()\` chains | ${thenChains} |`, + `| Migration trend (async / (async + callback)) | ${asyncFunctions}/${asyncFunctions + callbackFunctions} (${migrationPercent}%) |`, + '', + ].join('\n'), + ); // Output benchmark JSON for visualization const benchmarkData = [ @@ -113,7 +115,7 @@ if (process.env.GITHUB_STEP_SUMMARY) { name: 'Total callback functions', unit: 'count', value: callbackFunctions, - } + }, ]; writeFileSync('async-migration-benchmark.json', JSON.stringify(benchmarkData, null, 2)); } diff --git a/.github/workflows/alerts.yaml b/.github/workflows/alerts.yaml index 5713ac437e..71bdf56030 100644 --- a/.github/workflows/alerts.yaml +++ b/.github/workflows/alerts.yaml @@ -14,7 +14,7 @@ jobs: tests: - name: 1 minute interval tests file: monitoring/alerts.test.yaml - + - name: 10 seconds interval tests file: monitoring/alerts.10s.test.yaml diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index c81a539afe..ad63cecfb0 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -69,8 +69,7 @@ jobs: push: true context: images/federation provenance: false - build-args: - CLOUDSERVER_VERSION=${{ github.event.inputs.tag }} + build-args: CLOUDSERVER_VERSION=${{ github.event.inputs.tag }} tags: | ghcr.io/${{ github.repository }}:${{ github.event.inputs.tag }}-federation labels: | diff --git a/.nycrc b/.nycrc index 84e748de7c..bdd7c0a39f 100644 --- a/.nycrc +++ b/.nycrc @@ -1,10 +1,5 @@ { "all": true, - "include": [ - "bin/**/*.js", - "lib/**/*.js" - ], - "exclude": [ - "tests/**/*.js" - ] -} \ No newline at end of file + "include": ["bin/**/*.js", "lib/**/*.js"], + "exclude": ["tests/**/*.js"] +} diff --git a/CLAUDE.md b/CLAUDE.md index f96dcb50db..53cbb1fdaa 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -97,23 +97,23 @@ implementations: ### Data Backends (`S3DATA`) -| Backend | Port | Description | -|---------|------|-------------| -| `file` | 9991 | Local filesystem via `dataserver.js` | -| `multiple` | - | Multi-backend gateway (AWS S3, Azure, GCP, Sproxyd) | -| `mem` | - | In-memory (testing only) | +| Backend | Port | Description | +| ---------- | ---- | --------------------------------------------------- | +| `file` | 9991 | Local filesystem via `dataserver.js` | +| `multiple` | - | Multi-backend gateway (AWS S3, Azure, GCP, Sproxyd) | +| `mem` | - | In-memory (testing only) | `scality` is an alias for `multiple`. With `multiple`, objects route to backends defined in `locationConfig.json` based on location constraints. ### Metadata Backends (`S3METADATA`) -| Backend | Port | Description | -|---------|------|-------------| -| `file` (default) | 9990 | Local LevelDB via `mdserver.js` | -| `scality` | 9000 | External bucketd service (production Scality RING) | -| `mongodb` | 27017+ | MongoDB replica set | -| `mem` | - | In-memory (testing only) | +| Backend | Port | Description | +| ---------------- | ------ | -------------------------------------------------- | +| `file` (default) | 9990 | Local LevelDB via `mdserver.js` | +| `scality` | 9000 | External bucketd service (production Scality RING) | +| `mongodb` | 27017+ | MongoDB replica set | +| `mem` | - | In-memory (testing only) | **file vs scality**: The `file` backend runs a self-contained metadata server (`mdserver.js`) for development. The `scality` backend connects to external @@ -122,20 +122,20 @@ HA deployments. ### Auth Backends (`S3VAULT`) -| Backend | Port | Description | -|---------|------|-------------| -| `mem` (default) | - | In-memory accounts from `conf/authdata.json` | -| `vault` | 8500 | External Vault IAM service (vaultd) | +| Backend | Port | Description | +| --------------- | ---- | -------------------------------------------- | +| `mem` (default) | - | In-memory accounts from `conf/authdata.json` | +| `vault` | 8500 | External Vault IAM service (vaultd) | ### KMS Backends (`S3KMS`) -| Backend | Description | -|---------|-------------| +| Backend | Description | +| ---------------- | ---------------------------- | | `file` (default) | Local file-based key storage | -| `mem` | In-memory (testing only) | -| `kmip` | External KMIP server | -| `aws` | AWS KMS | -| `scality` | Scality KMS | +| `mem` | In-memory (testing only) | +| `kmip` | External KMIP server | +| `aws` | AWS KMS | +| `scality` | Scality KMS | ### Path Configuration diff --git a/Healthchecks.md b/Healthchecks.md index 3950c8bf29..95372bf23e 100644 --- a/Healthchecks.md +++ b/Healthchecks.md @@ -6,19 +6,19 @@ response with HTTP code - 200 OK - Server is up and running +Server is up and running - 500 Internal Server error - Server is experiencing an Internal Error +Server is experiencing an Internal Error - 400 Bad Request - Bad Request due to unsupported HTTP methods +Bad Request due to unsupported HTTP methods - 403 Forbidden - Request is not allowed due to IP restriction +Request is not allowed due to IP restriction ## Stats @@ -53,12 +53,12 @@ returned. This is accomplished by retrieving the 6 keys that represent the 6 five-second intervals. As Redis does not have a performant RANGE query, the list of keys are built manually as follows -* Take current timestamp +- Take current timestamp -* Build each key by subtracting the interval from the timestamp (5 seconds) +- Build each key by subtracting the interval from the timestamp (5 seconds) -* Total keys for each metric (total requests, 500s etc.) is TTL / interval - 30/5 = 6 +- Total keys for each metric (total requests, 500s etc.) is TTL / interval + 30/5 = 6 Note: When Redis is queried, results from non-existent keys are set to 0. diff --git a/TESTING.md b/TESTING.md index 68b8ff7a08..94f48527e0 100644 --- a/TESTING.md +++ b/TESTING.md @@ -10,26 +10,26 @@ ### Features tested - Authentication - - Building signature - - Checking timestamp - - Canonicalization - - Error Handling + - Building signature + - Checking timestamp + - Canonicalization + - Error Handling - Bucket Metadata API - - GET, PUT, DELETE Bucket Metadata + - GET, PUT, DELETE Bucket Metadata - s3 API - - GET Service - - GET, PUT, DELETE, HEAD Object - - GET, PUT, DELETE, HEAD Bucket - - ACL's - - Bucket Policies - - Lifecycle - - Range requests - - Multi-part upload + - GET Service + - GET, PUT, DELETE, HEAD Object + - GET, PUT, DELETE, HEAD Bucket + - ACL's + - Bucket Policies + - Lifecycle + - Range requests + - Multi-part upload - Routes - - GET, PUT, PUTRAW, DELETE, HEAD for objects and buckets + - GET, PUT, PUTRAW, DELETE, HEAD for objects and buckets ## Functional Tests diff --git a/bin/metrics_server.js b/bin/metrics_server.js index 752eb911f8..a44cf517fc 100755 --- a/bin/metrics_server.js +++ b/bin/metrics_server.js @@ -1,18 +1,11 @@ #!/usr/bin/env node 'use strict'; -const { - startWSManagementClient, - startPushConnectionHealthCheckServer, -} = require('../lib/management/push'); +const { startWSManagementClient, startPushConnectionHealthCheckServer } = require('../lib/management/push'); const logger = require('../lib/utilities/logger'); -const { - PUSH_ENDPOINT: pushEndpoint, - INSTANCE_ID: instanceId, - MANAGEMENT_TOKEN: managementToken, -} = process.env; +const { PUSH_ENDPOINT: pushEndpoint, INSTANCE_ID: instanceId, MANAGEMENT_TOKEN: managementToken } = process.env; if (!pushEndpoint) { logger.error('missing push endpoint env var'); diff --git a/bin/search_bucket.js b/bin/search_bucket.js index 14a14b0378..1d566a0724 100755 --- a/bin/search_bucket.js +++ b/bin/search_bucket.js @@ -9,15 +9,7 @@ const http = require('http'); const https = require('https'); const logger = require('../lib/utilities/logger'); -function _performSearch(host, - port, - bucketName, - query, - listVersions, - accessKey, - secretKey, - sessionToken, - verbose, ssl) { +function _performSearch(host, port, bucketName, query, listVersions, accessKey, secretKey, sessionToken, verbose, ssl) { const escapedSearch = encodeURIComponent(query); const options = { host, @@ -88,12 +80,13 @@ function searchBucket() { .option('-h, --host ', 'Host of the server') .option('-p, --port ', 'Port of the server') .option('-s', '--ssl', 'Enable ssl') - .option('-l, --list-versions', 'List all versions of the objects that meet the search query, ' + - 'otherwise only list the latest version') + .option( + '-l, --list-versions', + 'List all versions of the objects that meet the search query, ' + 'otherwise only list the latest version', + ) .option('-v, --verbose') .parse(process.argv); - const { host, port, accessKey, secretKey, sessionToken, bucket, query, listVersions, verbose, ssl } = - commander; + const { host, port, accessKey, secretKey, sessionToken, bucket, query, listVersions, verbose, ssl } = commander; if (!host || !port || !accessKey || !secretKey || !bucket || !query) { logger.error('missing parameter'); @@ -101,8 +94,7 @@ function searchBucket() { process.exit(1); } - _performSearch(host, port, bucket, query, listVersions, accessKey, secretKey, sessionToken, verbose, - ssl); + _performSearch(host, port, bucket, query, listVersions, accessKey, secretKey, sessionToken, verbose, ssl); } searchBucket(); diff --git a/bin/secure_channel_proxy.js b/bin/secure_channel_proxy.js index 165bf94c74..8b235f7fe0 100755 --- a/bin/secure_channel_proxy.js +++ b/bin/secure_channel_proxy.js @@ -1,18 +1,11 @@ #!/usr/bin/env node 'use strict'; -const { - startWSManagementClient, - startPushConnectionHealthCheckServer, -} = require('../lib/management/push'); +const { startWSManagementClient, startPushConnectionHealthCheckServer } = require('../lib/management/push'); const logger = require('../lib/utilities/logger'); -const { - PUSH_ENDPOINT: pushEndpoint, - INSTANCE_ID: instanceId, - MANAGEMENT_TOKEN: managementToken, -} = process.env; +const { PUSH_ENDPOINT: pushEndpoint, INSTANCE_ID: instanceId, MANAGEMENT_TOKEN: managementToken } = process.env; if (!pushEndpoint) { logger.error('missing push endpoint env var'); diff --git a/codecov.yml b/codecov.yml index ee3ed8cfa2..b1e83b5500 100644 --- a/codecov.yml +++ b/codecov.yml @@ -10,8 +10,8 @@ parsers: enable_partials: yes comment: - layout: newheader, reach, files, components, diff, flags # show component info in the PR comment - hide_comment_details: true # hide the comment details (e.g. coverage targets) in the PR comment + layout: newheader, reach, files, components, diff, flags # show component info in the PR comment + hide_comment_details: true # hide the comment details (e.g. coverage targets) in the PR comment # Setting coverage targets coverage: @@ -29,11 +29,11 @@ coverage: target: 80% component_management: - default_rules: # default rules that will be inherited by all components + default_rules: # default rules that will be inherited by all components statuses: [] flag_management: - default_rules: # the rules that will be followed for any flag added, generally + default_rules: # the rules that will be followed for any flag added, generally carryforward: true statuses: [] diff --git a/conf/authdata.json b/conf/authdata.json index 8cd34421c3..5f58ebde09 100644 --- a/conf/authdata.json +++ b/conf/authdata.json @@ -1,56 +1,69 @@ { - "accounts": [{ - "name": "Bart", - "email": "sampleaccount1@sampling.com", - "arn": "arn:aws:iam::123456789012:root", - "canonicalID": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", - "shortid": "123456789012", - "keys": [{ - "access": "accessKey1", - "secret": "verySecretKey1" - }] - }, { - "name": "Lisa", - "email": "sampleaccount2@sampling.com", - "arn": "arn:aws:iam::123456789013:root", - "canonicalID": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2bf", - "shortid": "123456789013", - "keys": [{ - "access": "accessKey2", - "secret": "verySecretKey2" - }] - }, - { - "name": "Clueso", - "email": "inspector@clueso.info", - "arn": "arn:aws:iam::123456789014:root", - "canonicalID": "http://acs.zenko.io/accounts/service/clueso", - "shortid": "123456789014", - "keys": [{ - "access": "cluesoKey1", - "secret": "cluesoSecretKey1" - }] - }, - { - "name": "Replication", - "email": "inspector@replication.info", - "arn": "arn:aws:iam::123456789015:root", - "canonicalID": "http://acs.zenko.io/accounts/service/replication", - "shortid": "123456789015", - "keys": [{ - "access": "replicationKey1", - "secret": "replicationSecretKey1" - }] - }, - { - "name": "Lifecycle", - "email": "inspector@lifecycle.info", - "arn": "arn:aws:iam::123456789016:root", - "canonicalID": "http://acs.zenko.io/accounts/service/lifecycle", - "shortid": "123456789016", - "keys": [{ - "access": "lifecycleKey1", - "secret": "lifecycleSecretKey1" - }] - }] + "accounts": [ + { + "name": "Bart", + "email": "sampleaccount1@sampling.com", + "arn": "arn:aws:iam::123456789012:root", + "canonicalID": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", + "shortid": "123456789012", + "keys": [ + { + "access": "accessKey1", + "secret": "verySecretKey1" + } + ] + }, + { + "name": "Lisa", + "email": "sampleaccount2@sampling.com", + "arn": "arn:aws:iam::123456789013:root", + "canonicalID": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2bf", + "shortid": "123456789013", + "keys": [ + { + "access": "accessKey2", + "secret": "verySecretKey2" + } + ] + }, + { + "name": "Clueso", + "email": "inspector@clueso.info", + "arn": "arn:aws:iam::123456789014:root", + "canonicalID": "http://acs.zenko.io/accounts/service/clueso", + "shortid": "123456789014", + "keys": [ + { + "access": "cluesoKey1", + "secret": "cluesoSecretKey1" + } + ] + }, + { + "name": "Replication", + "email": "inspector@replication.info", + "arn": "arn:aws:iam::123456789015:root", + "canonicalID": "http://acs.zenko.io/accounts/service/replication", + "shortid": "123456789015", + "keys": [ + { + "access": "replicationKey1", + "secret": "replicationSecretKey1" + } + ] + }, + { + "name": "Lifecycle", + "email": "inspector@lifecycle.info", + "arn": "arn:aws:iam::123456789016:root", + "canonicalID": "http://acs.zenko.io/accounts/service/lifecycle", + "shortid": "123456789016", + "keys": [ + { + "access": "lifecycleKey1", + "secret": "lifecycleSecretKey1" + } + ] + } + ] } diff --git a/config.json b/config.json index 1a6faf973a..abfd5ca93a 100644 --- a/config.json +++ b/config.json @@ -14,29 +14,34 @@ "zenko-cloudserver-replicator": "us-east-1", "lb": "us-east-1" }, - "websiteEndpoints": ["s3-website-us-east-1.amazonaws.com", - "s3-website.us-east-2.amazonaws.com", - "s3-website-us-west-1.amazonaws.com", - "s3-website-us-west-2.amazonaws.com", - "s3-website.ap-south-1.amazonaws.com", - "s3-website.ap-northeast-2.amazonaws.com", - "s3-website-ap-southeast-1.amazonaws.com", - "s3-website-ap-southeast-2.amazonaws.com", - "s3-website-ap-northeast-1.amazonaws.com", - "s3-website.eu-central-1.amazonaws.com", - "s3-website-eu-west-1.amazonaws.com", - "s3-website-sa-east-1.amazonaws.com", - "s3-website.localhost", - "s3-website.scality.test", - "zenkoazuretest.blob.core.windows.net"], - "replicationEndpoints": [{ - "site": "zenko", - "servers": ["127.0.0.1:8000"], - "default": true - }, { - "site": "us-east-2", - "type": "aws_s3" - }], + "websiteEndpoints": [ + "s3-website-us-east-1.amazonaws.com", + "s3-website.us-east-2.amazonaws.com", + "s3-website-us-west-1.amazonaws.com", + "s3-website-us-west-2.amazonaws.com", + "s3-website.ap-south-1.amazonaws.com", + "s3-website.ap-northeast-2.amazonaws.com", + "s3-website-ap-southeast-1.amazonaws.com", + "s3-website-ap-southeast-2.amazonaws.com", + "s3-website-ap-northeast-1.amazonaws.com", + "s3-website.eu-central-1.amazonaws.com", + "s3-website-eu-west-1.amazonaws.com", + "s3-website-sa-east-1.amazonaws.com", + "s3-website.localhost", + "s3-website.scality.test", + "zenkoazuretest.blob.core.windows.net" + ], + "replicationEndpoints": [ + { + "site": "zenko", + "servers": ["127.0.0.1:8000"], + "default": true + }, + { + "site": "us-east-2", + "type": "aws_s3" + } + ], "backbeat": { "host": "localhost", "port": 8900 @@ -135,7 +140,7 @@ "kmsHideScalityArn": false, "kmsAWS": { "providerName": "aws", - "region": "us-east-1", + "region": "us-east-1", "endpoint": "http://127.0.0.1:8080", "ak": "tbd", "sk": "tbd" diff --git a/constants.js b/constants.js index 1337f593e8..5e9ad2fe66 100644 --- a/constants.js +++ b/constants.js @@ -46,8 +46,7 @@ const constants = { // only public resources publicId: 'http://acs.amazonaws.com/groups/global/AllUsers', // All Authenticated Users is an ACL group. - allAuthedUsersId: 'http://acs.amazonaws.com/groups/' + - 'global/AuthenticatedUsers', + allAuthedUsersId: 'http://acs.amazonaws.com/groups/' + 'global/AuthenticatedUsers', // LogId is used for the AWS logger to write the logs // to the destination bucket. This style of logging is // to be implemented later but the logId is used in the @@ -74,8 +73,7 @@ const constants = { // Max size on put part or copy part is 5GB. For functional // testing use 110 MB as max - maximumAllowedPartSize: process.env.MPU_TESTING === 'yes' ? 110100480 : - 5368709120, + maximumAllowedPartSize: process.env.MPU_TESTING === 'yes' ? 110100480 : 5368709120, // Max size allowed in a single put object request is 5GB // https://docs.aws.amazon.com/AmazonS3/latest/dev/UploadingObjects.html @@ -101,15 +99,14 @@ const constants = { defaultApiBodySizeLimits: { // Multi Objects Delete request can be large : up to 1000 keys of 1024 bytes is // already 1mb, with the other fields it could reach 2mb - 'multiObjectDelete': 2 * 1024 * 1024, + multiObjectDelete: 2 * 1024 * 1024, // AWS sets the maximum size for bucket policies to 20 KB // https://docs.aws.amazon.com/AmazonS3/latest/userguide/add-bucket-policy.html - 'bucketPutPolicy': 20 * 1024, + bucketPutPolicy: 20 * 1024, }, // hex digest of sha256 hash of empty string: - emptyStringHash: crypto.createHash('sha256') - .update('', 'binary').digest('hex'), + emptyStringHash: crypto.createHash('sha256').update('', 'binary').digest('hex'), // Queries supported by AWS that we do not currently support. // Non-bucket queries @@ -164,8 +161,7 @@ const constants = { /* eslint-enable camelcase */ mpuMDStoredOnS3Backend: { azure: true }, azureAccountNameRegex: /^[a-z0-9]{3,24}$/, - base64Regex: new RegExp('^(?:[A-Za-z0-9+/]{4})*' + - '(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$'), + base64Regex: new RegExp('^(?:[A-Za-z0-9+/]{4})*' + '(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$'), productName: 'APN/1.0 Scality/1.0 Scality CloudServer for Zenko', // location constraint delimiter zenkoSeparator: ':', @@ -203,32 +199,15 @@ const constants = { invalidObjectUserMetadataHeader: 'x-amz-missing-meta', // Bucket specific queries supported by AWS that we do not currently support // these queries may or may not be supported at object level - unsupportedBucketQueries: [ - ], - suppressedUtapiEventFields: [ - 'object', - 'location', - 'versionId', - ], - allowedUtapiEventFilterFields: [ - 'operationId', - 'location', - 'account', - 'user', - 'bucket', - ], - arrayOfAllowed: [ - 'objectPutTagging', - 'objectPutLegalHold', - 'objectPutRetention', - ], + unsupportedBucketQueries: [], + suppressedUtapiEventFields: ['object', 'location', 'versionId'], + allowedUtapiEventFilterFields: ['operationId', 'location', 'account', 'user', 'bucket'], + arrayOfAllowed: ['objectPutTagging', 'objectPutLegalHold', 'objectPutRetention'], allowedUtapiEventFilterStates: ['allow', 'deny'], allowedRestoreObjectRequestTierValues: ['Standard'], // Only STANDARD class is supported, but keep the option to override supported values for now. // This should be removed in CLDSRV-639. - validStorageClasses: process.env.VALID_STORAGE_CLASSES?.split(',') || [ - 'STANDARD', - ], + validStorageClasses: process.env.VALID_STORAGE_CLASSES?.split(',') || ['STANDARD'], lifecycleListing: { CURRENT_TYPE: 'current', NON_CURRENT_TYPE: 'noncurrent', @@ -261,11 +240,7 @@ const constants = { assumedRoleArnResourceType: 'assumed-role', // Session name of the backbeat lifecycle assumed role session. backbeatLifecycleSessionName: 'backbeat-lifecycle', - actionsToConsiderAsObjectPut: [ - 'initiateMultipartUpload', - 'objectPutPart', - 'completeMultipartUpload', - ], + actionsToConsiderAsObjectPut: ['initiateMultipartUpload', 'objectPutPart', 'completeMultipartUpload'], // if requester is not bucket owner, bucket policy actions should be denied with // MethodNotAllowed error onlyOwnerAllowed: [ @@ -280,13 +255,7 @@ const constants = { rateLimitDefaultBurstCapacity: 1, rateLimitCleanupInterval: 10000, // 10 seconds // Supported attributes for the GetObjectAttributes 'x-amz-optional-attributes' header. - supportedGetObjectAttributes: new Set([ - 'StorageClass', - 'ObjectSize', - 'ObjectParts', - 'Checksum', - 'ETag', - ]), + supportedGetObjectAttributes: new Set(['StorageClass', 'ObjectSize', 'ObjectParts', 'Checksum', 'ETag']), }; module.exports = constants; diff --git a/dataserver.js b/dataserver.js index f1321422fb..f6bcabe481 100644 --- a/dataserver.js +++ b/dataserver.js @@ -14,9 +14,10 @@ process.on('uncaughtException', err => { process.exit(1); }); -if (config.backends.data === 'file' || - (config.backends.data === 'multiple' && - config.backends.metadata !== 'scality')) { +if ( + config.backends.data === 'file' || + (config.backends.data === 'multiple' && config.backends.metadata !== 'scality') +) { const dataServer = new arsenal.network.rest.RESTServer({ bindAddress: config.dataDaemon.bindAddress, port: config.dataDaemon.port, @@ -30,8 +31,7 @@ if (config.backends.data === 'file' || }); dataServer.setup(err => { if (err) { - logger.error('Error initializing REST data server', - { error: err }); + logger.error('Error initializing REST data server', { error: err }); return; } dataServer.start(); diff --git a/docs/OBJECT_LOCK_TEST_PLAN.md b/docs/OBJECT_LOCK_TEST_PLAN.md index 2c97ae5666..5072451d08 100644 --- a/docs/OBJECT_LOCK_TEST_PLAN.md +++ b/docs/OBJECT_LOCK_TEST_PLAN.md @@ -21,7 +21,7 @@ the new API actions. ### putBucket tests - passing option to enable object lock updates bucket metadata and enables - bucket versioning + bucket versioning ### putBucketVersioning tests @@ -43,17 +43,17 @@ the new API actions. ### initiateMultipartUpload tests - mpu object initiated with retention information should include retention - information + information ### putObjectLockConfiguration tests - putting configuration as non-bucket-owner user returns AccessDenied error - disabling object lock on bucket created with object lock returns error - enabling object lock on bucket created without object lock returns - InvalidBucketState error + InvalidBucketState error - enabling object lock with token on bucket created without object lock succeeds - putting valid object lock configuration when bucket does not have object - lock enabled returns error (InvalidRequest?) + lock enabled returns error (InvalidRequest?) - putting valid object lock configuration updates bucket metadata - putting invalid object lock configuration returns error - ObjectLockEnabled !== "Enabled" @@ -66,35 +66,35 @@ the new API actions. - getting configuration as non-bucket-owner user returns AccessDenied error - getting configuration when none is set returns - ObjectLockConfigurationNotFoundError error + ObjectLockConfigurationNotFoundError error - getting configuration returns correct object lock configuration for bucket ### putObjectRetention - putting retention as non-bucket-owner user returns AccessDenied error - putting retention on object in bucket without object lock enabled returns - InvalidRequest error + InvalidRequest error - putting valid retention period updates object metadata ### getObjectRetention - getting retention as non-bucket-owner user returns AccessDenied error - getting retention when none is set returns NoSuchObjectLockConfiguration - error + error - getting retention returns correct object retention period ### putObjectLegalHold - putting legal hold as non-bucket-owner user returns AccessDenied error - putting legal hold on object in bucket without object lock enabled returns - InvalidRequest error + InvalidRequest error - putting valid legal hold updates object metadata ### getObjectLegalHold - getting legal hold as non-bucket-owner user returns AccessDenied error - getting legal hold when none is set returns NoSuchObjectLockConfiguration - error + error - getting legal hold returns correct object legal hold ## End to End Tests @@ -102,22 +102,22 @@ the new API actions. ### Scenarios - Create bucket with object lock enabled. Put object. Put object lock - configuration. Put another object. + configuration. Put another object. - Ensure object put before configuration does not have retention period set - Ensure object put after configuration does have retention period set - Create bucket without object lock. Put object. Enable object lock with token - and put object lock configuration. Put another object. + and put object lock configuration. Put another object. - Ensure object put before configuration does not have retention period set - Ensure object put after configuration does have retention period set - Create bucket with object lock enabled and put configuration with COMPLIANCE - mode. Put object. + mode. Put object. - Ensure object cannot be deleted (returns AccessDenied error). - Ensure object cannot be overwritten. - Create bucket with object lock enabled and put configuration with GOVERNANCE - mode. Put object. + mode. Put object. - Ensure user without permission cannot delete object - Ensure user without permission cannot overwrite object - Ensure user with permission can delete object @@ -126,29 +126,29 @@ the new API actions. - Ensure user with permission cannot shorten retention period - Create bucket with object lock enabled and put configuration. Edit bucket - metadata so retention period is expired. Put object. + metadata so retention period is expired. Put object. - Ensure object can be deleted. - Ensure object can be overwritten. - Create bucket with object lock enabled and put configuration. Edit bucket - metadata so retention period is expired. Put object. Put new retention - period on object. + metadata so retention period is expired. Put object. Put new retention + period on object. - Ensure object cannot be deleted. - Ensure object cannot be overwritten. - Create bucket with object locked enabled and put configuration. Put object. - Edit object metadata so retention period is past expiration. + Edit object metadata so retention period is past expiration. - Ensure object can be deleted. - Ensure object can be overwritten. - Create bucket with object lock enabled and put configuration. Edit bucket - metadata so retention period is expired. Put object. Put legal hold - on object. + metadata so retention period is expired. Put object. Put legal hold + on object. - Ensure object cannot be deleted. - Ensure object cannot be overwritten. - Create bucket with object lock enabled and put configuration. Put object. - Check object retention. Change bucket object lock configuration. + Check object retention. Change bucket object lock configuration. - Ensure object retention period has not changed with bucket configuration. - Create bucket with object lock enabled. Put object with legal hold. @@ -156,6 +156,6 @@ the new API actions. - Ensure object cannot be overwritten. - Create bucket with object lock enabled. Put object with legal hold. Remove - legal hold. + legal hold. - Ensure object can be deleted. - Ensure object can be overwritten. diff --git a/docs/RELEASE.md b/docs/RELEASE.md index ddfb379180..87ab37070e 100644 --- a/docs/RELEASE.md +++ b/docs/RELEASE.md @@ -5,8 +5,8 @@ Docker images are hosted on [ghcri.io](https://github.com/orgs/scality/packages). CloudServer has a few images there: -* Cloudserver container image: ghcr.io/scality/cloudserver -* Dashboard oras image: ghcr.io/scality/cloudserver/cloudserver-dashboards +- Cloudserver container image: ghcr.io/scality/cloudserver +- Dashboard oras image: ghcr.io/scality/cloudserver/cloudserver-dashboards With every CI build, the CI will push images, tagging the content with the developer branch's short SHA-1 commit hash. @@ -26,38 +26,36 @@ docker pull ghcr.io/scality/cloudserver: To release a production image: -* Create a PR to bump the package version : +- Create a PR to bump the package version : update Cloudserver's `package.json` by bumping it to the relevant next version in a new PR. Per example if the last released version was `8.4.7`, the next version would be `8.4.8`. - ```js - { - "name": "cloudserver", - "version": "8.4.8", <--- Here - [...] - } - ``` - -* Review & merge the PR - -* Trigger the release workflow on GitHub - - * Go to the [**Actions** tab on GitHub](https://github.com/scality/cloudserver/actions) - * Select the `release` workflow from the list - * Click on **Run workflow** (manual dispatch) - * Enter the new tag (e.g., `8.4.8`) in the input field - * Start the workflow - - This workflow will create the tag and push the Docker images. - - This should be done as soon as the PR is merged, - so that the tag is put on the "version bump" commit. - -* Release the release version on Jira - - * Go to the [CloudServer release page](https://scality.atlassian.net/projects/CLDSRV?selectedItem=com.atlassian.jira.jira-projects-plugin:release-page) - * Create a next version - * Name: `[next version]`, in this example `8.4.9` - * Click `...` and select `Release` on the recently released version (`8.4.8`) - * Fill in the field to move incomplete version to the next one + ```js + { + "name": "cloudserver", + "version": "8.4.8", <--- Here + [...] + } + ``` + +- Review & merge the PR + +- Trigger the release workflow on GitHub + - Go to the [**Actions** tab on GitHub](https://github.com/scality/cloudserver/actions) + - Select the `release` workflow from the list + - Click on **Run workflow** (manual dispatch) + - Enter the new tag (e.g., `8.4.8`) in the input field + - Start the workflow + + This workflow will create the tag and push the Docker images. + + This should be done as soon as the PR is merged, + so that the tag is put on the "version bump" commit. + +- Release the release version on Jira + - Go to the [CloudServer release page](https://scality.atlassian.net/projects/CLDSRV?selectedItem=com.atlassian.jira.jira-projects-plugin:release-page) + - Create a next version + - Name: `[next version]`, in this example `8.4.9` + - Click `...` and select `Release` on the recently released version (`8.4.8`) + - Fill in the field to move incomplete version to the next one diff --git a/eslint.config.mjs b/eslint.config.mjs index 043f5b7f99..5bc5a1c391 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -1,78 +1,81 @@ -import mocha from "eslint-plugin-mocha"; -import promise from "eslint-plugin-promise"; -import n from "eslint-plugin-n"; -import path from "node:path"; -import { fileURLToPath } from "node:url"; -import js from "@eslint/js"; -import { FlatCompat } from "@eslint/eslintrc"; +import mocha from 'eslint-plugin-mocha'; +import promise from 'eslint-plugin-promise'; +import n from 'eslint-plugin-n'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import js from '@eslint/js'; +import { FlatCompat } from '@eslint/eslintrc'; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); const compat = new FlatCompat({ baseDirectory: __dirname, recommendedConfig: js.configs.recommended, - allConfig: js.configs.all + allConfig: js.configs.all, }); -export default [...compat.extends('@scality/scality'), { - plugins: { - mocha, - promise, - n, - }, +export default [ + ...compat.extends('@scality/scality'), + { + plugins: { + mocha, + promise, + n, + }, - languageOptions: { - ecmaVersion: 2021, - sourceType: "script", - }, + languageOptions: { + ecmaVersion: 2021, + sourceType: 'script', + }, - rules: { - "import/extensions": "off", - "lines-around-directive": "off", - "no-underscore-dangle": "off", - "indent": "off", - "object-curly-newline": "off", - "operator-linebreak": "off", - "function-paren-newline": "off", - "import/newline-after-import": "off", - "prefer-destructuring": "off", - "implicit-arrow-linebreak": "off", - "no-bitwise": "off", - "dot-location": "off", - "comma-dangle": "off", - "no-undef-init": "off", - "global-require": "off", - "import/no-dynamic-require": "off", - "class-methods-use-this": "off", - "no-plusplus": "off", - "no-else-return": "off", - "object-property-newline": "off", - "import/order": "off", - "no-continue": "off", - "no-tabs": "off", - "lines-between-class-members": "off", - "prefer-spread": "off", - "no-lonely-if": "off", - "no-useless-escape": "off", - "no-restricted-globals": "off", - "no-buffer-constructor": "off", - "import/no-extraneous-dependencies": "off", - "space-unary-ops": "off", - "no-useless-return": "off", - "no-unexpected-multiline": "off", - "no-mixed-operators": "off", - "newline-per-chained-call": "off", - "operator-assignment": "off", - "spaced-comment": "off", - "comma-style": "off", - "no-restricted-properties": "off", - "new-parens": "off", - "no-multi-spaces": "off", - "quote-props": "off", - "max-len": "off", - "mocha/no-exclusive-tests": "error", - "no-redeclare": ["error", { "builtinGlobals": false }], - "promise/prefer-await-to-then": "warn", - "n/callback-return": "warn", + rules: { + 'import/extensions': 'off', + 'lines-around-directive': 'off', + 'no-underscore-dangle': 'off', + indent: 'off', + 'object-curly-newline': 'off', + 'operator-linebreak': 'off', + 'function-paren-newline': 'off', + 'import/newline-after-import': 'off', + 'prefer-destructuring': 'off', + 'implicit-arrow-linebreak': 'off', + 'no-bitwise': 'off', + 'dot-location': 'off', + 'comma-dangle': 'off', + 'no-undef-init': 'off', + 'global-require': 'off', + 'import/no-dynamic-require': 'off', + 'class-methods-use-this': 'off', + 'no-plusplus': 'off', + 'no-else-return': 'off', + 'object-property-newline': 'off', + 'import/order': 'off', + 'no-continue': 'off', + 'no-tabs': 'off', + 'lines-between-class-members': 'off', + 'prefer-spread': 'off', + 'no-lonely-if': 'off', + 'no-useless-escape': 'off', + 'no-restricted-globals': 'off', + 'no-buffer-constructor': 'off', + 'import/no-extraneous-dependencies': 'off', + 'space-unary-ops': 'off', + 'no-useless-return': 'off', + 'no-unexpected-multiline': 'off', + 'no-mixed-operators': 'off', + 'newline-per-chained-call': 'off', + 'operator-assignment': 'off', + 'spaced-comment': 'off', + 'comma-style': 'off', + 'no-restricted-properties': 'off', + 'new-parens': 'off', + 'no-multi-spaces': 'off', + 'quote-props': 'off', + 'max-len': 'off', + 'mocha/no-exclusive-tests': 'error', + 'no-redeclare': ['error', { builtinGlobals: false }], + 'promise/prefer-await-to-then': 'warn', + 'n/callback-return': 'warn', + }, }, -}]; +]; diff --git a/examples/node-md-search.js b/examples/node-md-search.js index d660e44fb1..54068795fc 100644 --- a/examples/node-md-search.js +++ b/examples/node-md-search.js @@ -32,12 +32,13 @@ command.middlewareStack.add( { step: 'build', name: 'addSearchParameter', - priority: 'high' - } + priority: 'high', + }, ); // Send command and handle response -s3Client.send(command) +s3Client + .send(command) .then(data => { process.stdout.write(`Result ${JSON.stringify(data)}`); }) diff --git a/lib/Config.js b/lib/Config.js index 51e452fa18..72828fa054 100644 --- a/lib/Config.js +++ b/lib/Config.js @@ -35,10 +35,7 @@ const { const { parseRateLimitConfig } = require('./api/apiUtils/rateLimit/config'); // config paths -const configSearchPaths = [ - path.join(__dirname, '../conf'), - path.join(__dirname, '..'), -]; +const configSearchPaths = [path.join(__dirname, '../conf'), path.join(__dirname, '..')]; function findConfigFile(fileName) { if (fileName[0] === '/') { @@ -49,8 +46,10 @@ function findConfigFile(fileName) { return fs.existsSync(testFilePath); }); if (!containingPath) { - throw new Error(`Unable to find the configuration file "${fileName}" ` + - `under the paths: ${JSON.stringify(configSearchPaths)}`); + throw new Error( + `Unable to find the configuration file "${fileName}" ` + + `under the paths: ${JSON.stringify(configSearchPaths)}`, + ); } return path.join(containingPath, fileName); } @@ -85,26 +84,26 @@ function assertCertPaths(key, cert, ca, basePath) { certObj.certs = {}; if (key) { const keypath = key.startsWith('/') ? key : `${basePath}/${key}`; - assert.doesNotThrow(() => - fs.accessSync(keypath, fs.F_OK | fs.R_OK), - `File not found or unreachable: ${keypath}`); + assert.doesNotThrow( + () => fs.accessSync(keypath, fs.F_OK | fs.R_OK), + `File not found or unreachable: ${keypath}`, + ); certObj.paths.key = keypath; certObj.certs.key = fs.readFileSync(keypath, 'ascii'); } if (cert) { const certpath = cert.startsWith('/') ? cert : `${basePath}/${cert}`; - assert.doesNotThrow(() => - fs.accessSync(certpath, fs.F_OK | fs.R_OK), - `File not found or unreachable: ${certpath}`); + assert.doesNotThrow( + () => fs.accessSync(certpath, fs.F_OK | fs.R_OK), + `File not found or unreachable: ${certpath}`, + ); certObj.paths.cert = certpath; certObj.certs.cert = fs.readFileSync(certpath, 'ascii'); } if (ca) { const capath = ca.startsWith('/') ? ca : `${basePath}/${ca}`; - assert.doesNotThrow(() => - fs.accessSync(capath, fs.F_OK | fs.R_OK), - `File not found or unreachable: ${capath}`); + assert.doesNotThrow(() => fs.accessSync(capath, fs.F_OK | fs.R_OK), `File not found or unreachable: ${capath}`); certObj.paths.ca = capath; certObj.certs.ca = fs.readFileSync(capath, 'ascii'); } @@ -121,48 +120,56 @@ function parseSproxydConfig(configSproxyd) { } function parseRedisConfig(redisConfig) { - const joiSchema = joi.object({ - password: joi.string().allow(''), - host: joi.string(), - port: joi.number(), - retry: joi.object({ - connectBackoff: joi.object({ - min: joi.number().required(), - max: joi.number().required(), - jitter: joi.number().required(), - factor: joi.number().required(), - deadline: joi.number().required(), + const joiSchema = joi + .object({ + password: joi.string().allow(''), + host: joi.string(), + port: joi.number(), + retry: joi.object({ + connectBackoff: joi.object({ + min: joi.number().required(), + max: joi.number().required(), + jitter: joi.number().required(), + factor: joi.number().required(), + deadline: joi.number().required(), + }), }), - }), - // sentinel config - sentinels: joi.alternatives().try( - joi.string() - .pattern(/^[a-zA-Z0-9.-]+:[0-9]+(,[a-zA-Z0-9.-]+:[0-9]+)*$/) - .custom(hosts => hosts.split(',').map(item => { - const [host, port] = item.split(':'); - return { host, port: Number.parseInt(port, 10) }; - })), - joi.array().items( - joi.object({ - host: joi.string().required(), - port: joi.number().required(), - }) - ).min(1), - ), - name: joi.string(), - sentinelPassword: joi.string().allow(''), - }) - .and('host', 'port') - .and('sentinels', 'name') - .xor('host', 'sentinels') - .without('sentinels', ['host', 'port']) - .without('host', ['sentinels', 'sentinelPassword']); + // sentinel config + sentinels: joi.alternatives().try( + joi + .string() + .pattern(/^[a-zA-Z0-9.-]+:[0-9]+(,[a-zA-Z0-9.-]+:[0-9]+)*$/) + .custom(hosts => + hosts.split(',').map(item => { + const [host, port] = item.split(':'); + return { host, port: Number.parseInt(port, 10) }; + }), + ), + joi + .array() + .items( + joi.object({ + host: joi.string().required(), + port: joi.number().required(), + }), + ) + .min(1), + ), + name: joi.string(), + sentinelPassword: joi.string().allow(''), + }) + .and('host', 'port') + .and('sentinels', 'name') + .xor('host', 'sentinels') + .without('sentinels', ['host', 'port']) + .without('host', ['sentinels', 'sentinelPassword']); return joi.attempt(redisConfig, joiSchema, 'bad config'); } function parseSupportedLifecycleRules(supportedLifecycleRulesConfig) { - const supportedLifecycleRulesSchema = joi.array() + const supportedLifecycleRulesSchema = joi + .array() .items(joi.string().valid(...supportedLifecycleRules)) .default(supportedLifecycleRules) .min(1); @@ -174,52 +181,38 @@ function parseSupportedLifecycleRules(supportedLifecycleRulesConfig) { } function restEndpointsAssert(restEndpoints, locationConstraints) { - assert(typeof restEndpoints === 'object', - 'bad config: restEndpoints must be an object of endpoints'); - assert(Object.keys(restEndpoints).every( - r => typeof restEndpoints[r] === 'string'), - 'bad config: each endpoint must be a string'); - assert(Object.keys(restEndpoints).every( - r => typeof locationConstraints[restEndpoints[r]] === 'object'), - 'bad config: rest endpoint target not in locationConstraints'); + assert(typeof restEndpoints === 'object', 'bad config: restEndpoints must be an object of endpoints'); + assert( + Object.keys(restEndpoints).every(r => typeof restEndpoints[r] === 'string'), + 'bad config: each endpoint must be a string', + ); + assert( + Object.keys(restEndpoints).every(r => typeof locationConstraints[restEndpoints[r]] === 'object'), + 'bad config: rest endpoint target not in locationConstraints', + ); } function gcpLocationConstraintAssert(location, locationObj) { - const { - gcpEndpoint, - bucketName, - mpuBucketName, - } = locationObj.details; - const stringFields = [ - gcpEndpoint, - bucketName, - mpuBucketName, - ]; + const { gcpEndpoint, bucketName, mpuBucketName } = locationObj.details; + const stringFields = [gcpEndpoint, bucketName, mpuBucketName]; stringFields.forEach(field => { if (field !== undefined) { - assert(typeof field === 'string', - `bad config: ${field} must be a string`); + assert(typeof field === 'string', `bad config: ${field} must be a string`); } }); } function azureGetStorageAccountName(location, locationDetails) { const { azureStorageAccountName } = locationDetails; - const storageAccountNameFromEnv = - process.env[`${location}_AZURE_STORAGE_ACCOUNT_NAME`]; + const storageAccountNameFromEnv = process.env[`${location}_AZURE_STORAGE_ACCOUNT_NAME`]; return storageAccountNameFromEnv || azureStorageAccountName; } function azureGetLocationCredentials(location, locationDetails) { const storageAccessKey = - process.env[`${location}_AZURE_STORAGE_ACCESS_KEY`] || - locationDetails.azureStorageAccessKey; - const sasToken = - process.env[`${location}_AZURE_SAS_TOKEN`] || - locationDetails.sasToken; - const clientKey = - process.env[`${location}_AZURE_CLIENT_KEY`] || - locationDetails.clientKey; + process.env[`${location}_AZURE_STORAGE_ACCESS_KEY`] || locationDetails.azureStorageAccessKey; + const sasToken = process.env[`${location}_AZURE_SAS_TOKEN`] || locationDetails.sasToken; + const clientKey = process.env[`${location}_AZURE_CLIENT_KEY`] || locationDetails.clientKey; const authMethod = process.env[`${location}_AZURE_AUTH_METHOD`] || @@ -230,32 +223,27 @@ function azureGetLocationCredentials(location, locationDetails) { 'shared-key'; switch (authMethod) { - case 'shared-key': - default: - return { - authMethod, - storageAccountName: - azureGetStorageAccountName(location, locationDetails), - storageAccessKey, - }; + case 'shared-key': + default: + return { + authMethod, + storageAccountName: azureGetStorageAccountName(location, locationDetails), + storageAccessKey, + }; - case 'shared-access-signature': - return { - authMethod, - sasToken, - }; + case 'shared-access-signature': + return { + authMethod, + sasToken, + }; - case 'client-secret': - return { - authMethod, - tenantId: - process.env[`${location}_AZURE_TENANT_ID`] || - locationDetails.tenantId, - clientId: - process.env[`${location}_AZURE_CLIENT_ID`] || - locationDetails.clientId, - clientKey, - }; + case 'client-secret': + return { + authMethod, + tenantId: process.env[`${location}_AZURE_TENANT_ID`] || locationDetails.tenantId, + clientId: process.env[`${location}_AZURE_CLIENT_ID`] || locationDetails.clientId, + clientKey, + }; } } @@ -263,120 +251,128 @@ function azureLocationConstraintAssert(location, locationObj) { const locationParams = { ...azureGetLocationCredentials(location, locationObj.details), azureStorageEndpoint: - process.env[`${location}_AZURE_STORAGE_ENDPOINT`] || - locationObj.details.azureStorageEndpoint, + process.env[`${location}_AZURE_STORAGE_ENDPOINT`] || locationObj.details.azureStorageEndpoint, azureContainerName: locationObj.details.azureContainerName, }; Object.keys(locationParams).forEach(param => { const value = locationParams[param]; - assert.notEqual(value, undefined, + assert.notEqual( + value, + undefined, `bad location constraint: "${location}" ${param} ` + - 'must be set in locationConfig or environment variable'); - assert.strictEqual(typeof value, 'string', - `bad location constraint: "${location}" ${param} ` + - `"${value}" must be a string`); + 'must be set in locationConfig or environment variable', + ); + assert.strictEqual( + typeof value, + 'string', + `bad location constraint: "${location}" ${param} ` + `"${value}" must be a string`, + ); }); if (locationParams.authMethod === 'shared-key') { - assert(azureAccountNameRegex.test(locationParams.storageAccountName), + assert( + azureAccountNameRegex.test(locationParams.storageAccountName), `bad location constraint: "${location}" azureStorageAccountName ` + - `"${locationParams.storageAccountName}" is an invalid value`); - assert(base64Regex.test(locationParams.storageAccessKey), - `bad location constraint: "${location}" ` + - 'azureStorageAccessKey is not a valid base64 string'); + `"${locationParams.storageAccountName}" is an invalid value`, + ); + assert( + base64Regex.test(locationParams.storageAccessKey), + `bad location constraint: "${location}" ` + 'azureStorageAccessKey is not a valid base64 string', + ); } - assert(isValidBucketName(locationParams.azureContainerName, []), - `bad location constraint: "${location}" ` + - 'azureContainerName is an invalid container name'); + assert( + isValidBucketName(locationParams.azureContainerName, []), + `bad location constraint: "${location}" ` + 'azureContainerName is an invalid container name', + ); } function hdClientLocationConstraintAssert(configHd) { const hdclientFields = []; if (configHd.bootstrap !== undefined) { - assert(Array.isArray(configHd.bootstrap) - && configHd.bootstrap - .every(e => typeof e === 'string'), - 'bad config: hdclient.bootstrap must be an array of strings'); - assert(configHd.bootstrap.length > 0, - 'bad config: hdclient bootstrap list is empty'); + assert( + Array.isArray(configHd.bootstrap) && configHd.bootstrap.every(e => typeof e === 'string'), + 'bad config: hdclient.bootstrap must be an array of strings', + ); + assert(configHd.bootstrap.length > 0, 'bad config: hdclient bootstrap list is empty'); hdclientFields.push('bootstrap'); } return hdclientFields; } function locationConstraintAssert(locationConstraints) { - const supportedBackends = [ - 'mem', 'file', 'scality', 'mongodb', 'tlp', 'crr' - ].concat(Object.keys(validExternalBackends)); - assert(typeof locationConstraints === 'object', - 'bad config: locationConstraints must be an object'); + const supportedBackends = ['mem', 'file', 'scality', 'mongodb', 'tlp', 'crr'].concat( + Object.keys(validExternalBackends), + ); + assert(typeof locationConstraints === 'object', 'bad config: locationConstraints must be an object'); Object.keys(locationConstraints).forEach(l => { - assert(typeof locationConstraints[l] === 'object', - 'bad config: locationConstraints[region] must be an object'); - assert(typeof locationConstraints[l].type === 'string', - 'bad config: locationConstraints[region].type is ' + - 'mandatory and must be a string'); - assert(supportedBackends.indexOf(locationConstraints[l].type) > -1, - 'bad config: locationConstraints[region].type must ' + - `be one of ${supportedBackends}`); - assert(typeof locationConstraints[l].objectId === 'string', - 'bad config: locationConstraints[region].objectId is ' + - 'mandatory and must be a unique string across locations'); - assert(Object.keys(locationConstraints) - .filter(loc => (locationConstraints[loc].objectId === - locationConstraints[l].objectId)) - .length === 1, - 'bad config: location constraint objectId ' + - `"${locationConstraints[l].objectId}" is not unique across ` + - 'configured locations'); - assert(typeof locationConstraints[l].legacyAwsBehavior - === 'boolean', - 'bad config: locationConstraints[region]' + - '.legacyAwsBehavior is mandatory and must be a boolean'); - assert(['undefined', 'boolean'].includes( - typeof locationConstraints[l].isTransient), - 'bad config: locationConstraints[region]' + - '.isTransient must be a boolean'); + assert(typeof locationConstraints[l] === 'object', 'bad config: locationConstraints[region] must be an object'); + assert( + typeof locationConstraints[l].type === 'string', + 'bad config: locationConstraints[region].type is ' + 'mandatory and must be a string', + ); + assert( + supportedBackends.indexOf(locationConstraints[l].type) > -1, + 'bad config: locationConstraints[region].type must ' + `be one of ${supportedBackends}`, + ); + assert( + typeof locationConstraints[l].objectId === 'string', + 'bad config: locationConstraints[region].objectId is ' + + 'mandatory and must be a unique string across locations', + ); + assert( + Object.keys(locationConstraints).filter( + loc => locationConstraints[loc].objectId === locationConstraints[l].objectId, + ).length === 1, + 'bad config: location constraint objectId ' + + `"${locationConstraints[l].objectId}" is not unique across ` + + 'configured locations', + ); + assert( + typeof locationConstraints[l].legacyAwsBehavior === 'boolean', + 'bad config: locationConstraints[region]' + '.legacyAwsBehavior is mandatory and must be a boolean', + ); + assert( + ['undefined', 'boolean'].includes(typeof locationConstraints[l].isTransient), + 'bad config: locationConstraints[region]' + '.isTransient must be a boolean', + ); if (locationConstraints[l].sizeLimitGB !== undefined) { - assert(typeof locationConstraints[l].sizeLimitGB === 'number' || - locationConstraints[l].sizeLimitGB === null, - 'bad config: locationConstraints[region].sizeLimitGB ' + - 'must be a number (in gigabytes)'); + assert( + typeof locationConstraints[l].sizeLimitGB === 'number' || locationConstraints[l].sizeLimitGB === null, + 'bad config: locationConstraints[region].sizeLimitGB ' + 'must be a number (in gigabytes)', + ); } const details = locationConstraints[l].details; - assert(typeof details === 'object', - 'bad config: locationConstraints[region].details is ' + - 'mandatory and must be an object'); + assert( + typeof details === 'object', + 'bad config: locationConstraints[region].details is ' + 'mandatory and must be an object', + ); if (details.serverSideEncryption !== undefined) { - assert(typeof details.serverSideEncryption === 'boolean', - 'bad config: locationConstraints[region]' + - '.details.serverSideEncryption must be a boolean'); - } - const stringFields = [ - 'awsEndpoint', - 'bucketName', - 'credentialsProfile', - 'region', - ]; + assert( + typeof details.serverSideEncryption === 'boolean', + 'bad config: locationConstraints[region]' + '.details.serverSideEncryption must be a boolean', + ); + } + const stringFields = ['awsEndpoint', 'bucketName', 'credentialsProfile', 'region']; stringFields.forEach(field => { if (details[field] !== undefined) { - assert(typeof details[field] === 'string', - `bad config: ${field} must be a string`); + assert(typeof details[field] === 'string', `bad config: ${field} must be a string`); } }); if (details.bucketMatch !== undefined) { - assert(typeof details.bucketMatch === 'boolean', - 'bad config: details.bucketMatch must be a boolean'); + assert(typeof details.bucketMatch === 'boolean', 'bad config: details.bucketMatch must be a boolean'); } if (details.credentials !== undefined) { - assert(typeof details.credentials === 'object', - 'bad config: details.credentials must be an object'); - assert(typeof details.credentials.accessKey === 'string', - 'bad config: credentials must include accessKey as string'); - assert(typeof details.credentials.secretKey === 'string', - 'bad config: credentials must include secretKey as string'); + assert(typeof details.credentials === 'object', 'bad config: details.credentials must be an object'); + assert( + typeof details.credentials.accessKey === 'string', + 'bad config: credentials must include accessKey as string', + ); + assert( + typeof details.credentials.secretKey === 'string', + 'bad config: credentials must include secretKey as string', + ); } if (locationConstraints[l].type === 'tlp') { @@ -388,25 +384,30 @@ function locationConstraintAssert(locationConstraints) { } if (details.https !== undefined) { - assert(typeof details.https === 'boolean', 'bad config: ' + - 'locationConstraints[region].details https must be a boolean'); + assert( + typeof details.https === 'boolean', + 'bad config: ' + 'locationConstraints[region].details https must be a boolean', + ); } else { // eslint-disable-next-line no-param-reassign locationConstraints[l].details.https = true; } if (details.pathStyle !== undefined) { - assert(typeof details.pathStyle === 'boolean', 'bad config: ' + - 'locationConstraints[region].pathStyle must be a boolean'); + assert( + typeof details.pathStyle === 'boolean', + 'bad config: ' + 'locationConstraints[region].pathStyle must be a boolean', + ); } else { // eslint-disable-next-line no-param-reassign locationConstraints[l].details.pathStyle = false; } if (details.supportsVersioning !== undefined) { - assert(typeof details.supportsVersioning === 'boolean', - 'bad config: locationConstraints[region].supportsVersioning' + - 'must be a boolean'); + assert( + typeof details.supportsVersioning === 'boolean', + 'bad config: locationConstraints[region].supportsVersioning' + 'must be a boolean', + ); } else { // default to true // eslint-disable-next-line no-param-reassign @@ -420,52 +421,45 @@ function locationConstraintAssert(locationConstraints) { gcpLocationConstraintAssert(l, locationConstraints[l]); } if (locationConstraints[l].type === 'pfs') { - assert(typeof details.pfsDaemonEndpoint === 'object', - 'bad config: pfsDaemonEndpoint is mandatory and must be an object'); + assert( + typeof details.pfsDaemonEndpoint === 'object', + 'bad config: pfsDaemonEndpoint is mandatory and must be an object', + ); } - if (locationConstraints[l].type === 'scality' && + if ( + locationConstraints[l].type === 'scality' && locationConstraints[l].details.connector !== undefined && - locationConstraints[l].details.connector.hdclient !== undefined) { - hdClientLocationConstraintAssert( - locationConstraints[l].details.connector.hdclient); + locationConstraints[l].details.connector.hdclient !== undefined + ) { + hdClientLocationConstraintAssert(locationConstraints[l].details.connector.hdclient); } }); - assert(Object.keys(locationConstraints) - .includes('us-east-1'), 'bad locationConfig: must ' + - 'include us-east-1 as a locationConstraint'); + assert( + Object.keys(locationConstraints).includes('us-east-1'), + 'bad locationConfig: must ' + 'include us-east-1 as a locationConstraint', + ); } function parseUtapiReindex(config) { - const { - enabled, - schedule, - redis, - bucketd, - onlyCountLatestWhenObjectLocked, - } = config; - assert(typeof enabled === 'boolean', - 'bad config: utapi.reindex.enabled must be a boolean'); + const { enabled, schedule, redis, bucketd, onlyCountLatestWhenObjectLocked } = config; + assert(typeof enabled === 'boolean', 'bad config: utapi.reindex.enabled must be a boolean'); const parsedRedis = parseRedisConfig(redis); - assert(Array.isArray(parsedRedis.sentinels), - 'bad config: utapi reindex redis config requires a list of sentinels'); - - assert(typeof bucketd === 'object', - 'bad config: utapi.reindex.bucketd must be an object'); - assert(typeof bucketd.port === 'number', - 'bad config: utapi.reindex.bucketd.port must be a number'); - assert(typeof schedule === 'string', - 'bad config: utapi.reindex.schedule must be a string'); + assert(Array.isArray(parsedRedis.sentinels), 'bad config: utapi reindex redis config requires a list of sentinels'); + + assert(typeof bucketd === 'object', 'bad config: utapi.reindex.bucketd must be an object'); + assert(typeof bucketd.port === 'number', 'bad config: utapi.reindex.bucketd.port must be a number'); + assert(typeof schedule === 'string', 'bad config: utapi.reindex.schedule must be a string'); if (onlyCountLatestWhenObjectLocked !== undefined) { - assert(typeof onlyCountLatestWhenObjectLocked === 'boolean', - 'bad config: utapi.reindex.onlyCountLatestWhenObjectLocked must be a boolean'); + assert( + typeof onlyCountLatestWhenObjectLocked === 'boolean', + 'bad config: utapi.reindex.onlyCountLatestWhenObjectLocked must be a boolean', + ); } try { cronParser.parseExpression(schedule); } catch (e) { - assert(false, - 'bad config: utapi.reindex.schedule must be a valid ' + - `cron schedule. ${e.message}.`); + assert(false, 'bad config: utapi.reindex.schedule must be a valid ' + `cron schedule. ${e.message}.`); } return { enabled, @@ -478,61 +472,59 @@ function parseUtapiReindex(config) { function requestsConfigAssert(requestsConfig) { if (requestsConfig.viaProxy !== undefined) { - assert(typeof requestsConfig.viaProxy === 'boolean', - 'config: invalid requests configuration. viaProxy must be a ' + - 'boolean'); + assert( + typeof requestsConfig.viaProxy === 'boolean', + 'config: invalid requests configuration. viaProxy must be a ' + 'boolean', + ); if (requestsConfig.viaProxy) { - assert(Array.isArray(requestsConfig.trustedProxyCIDRs) && - requestsConfig.trustedProxyCIDRs.length > 0 && - requestsConfig.trustedProxyCIDRs - .every(ip => typeof ip === 'string'), - 'config: invalid requests configuration. ' + - 'trustedProxyCIDRs must be set if viaProxy is set to true ' + - 'and must be an array'); - - assert(typeof requestsConfig.extractClientIPFromHeader === 'string' - && requestsConfig.extractClientIPFromHeader.length > 0, - 'config: invalid requests configuration. ' + - 'extractClientIPFromHeader must be set if viaProxy is ' + - 'set to true and must be a string'); - - assert(typeof requestsConfig.extractProtocolFromHeader === 'string' - && requestsConfig.extractProtocolFromHeader.length > 0, - 'config: invalid requests configuration. ' + - 'extractProtocolFromHeader must be set if viaProxy is ' + - 'set to true and must be a string'); + assert( + Array.isArray(requestsConfig.trustedProxyCIDRs) && + requestsConfig.trustedProxyCIDRs.length > 0 && + requestsConfig.trustedProxyCIDRs.every(ip => typeof ip === 'string'), + 'config: invalid requests configuration. ' + + 'trustedProxyCIDRs must be set if viaProxy is set to true ' + + 'and must be an array', + ); + + assert( + typeof requestsConfig.extractClientIPFromHeader === 'string' && + requestsConfig.extractClientIPFromHeader.length > 0, + 'config: invalid requests configuration. ' + + 'extractClientIPFromHeader must be set if viaProxy is ' + + 'set to true and must be a string', + ); + + assert( + typeof requestsConfig.extractProtocolFromHeader === 'string' && + requestsConfig.extractProtocolFromHeader.length > 0, + 'config: invalid requests configuration. ' + + 'extractProtocolFromHeader must be set if viaProxy is ' + + 'set to true and must be a string', + ); } // All headers in NodeJS are lowercase: to be exploitable // we need to lowercase the value. // eslint-disable-next-line no-param-reassign - requestsConfig.extractClientIPFromHeader = - requestsConfig.extractClientIPFromHeader?.toLowerCase(); + requestsConfig.extractClientIPFromHeader = requestsConfig.extractClientIPFromHeader?.toLowerCase(); // eslint-disable-next-line no-param-reassign - requestsConfig.extractProtocolFromHeader = - requestsConfig.extractProtocolFromHeader?.toLowerCase(); + requestsConfig.extractProtocolFromHeader = requestsConfig.extractProtocolFromHeader?.toLowerCase(); } } function bucketNotifAssert(bucketNotifConfig) { - assert(Array.isArray(bucketNotifConfig), - 'bad config: bucket notification configuration must be an array'); + assert(Array.isArray(bucketNotifConfig), 'bad config: bucket notification configuration must be an array'); bucketNotifConfig.forEach(c => { const { resource, type, host, port, auth } = c; - assert(typeof resource === 'string', - 'bad config: bucket notification configuration resource must be a string'); - assert(typeof type === 'string', - 'bad config: bucket notification configuration type must be a string'); - assert(typeof host === 'string' && host !== '', - 'bad config: hostname must be a non-empty string'); + assert(typeof resource === 'string', 'bad config: bucket notification configuration resource must be a string'); + assert(typeof type === 'string', 'bad config: bucket notification configuration type must be a string'); + assert(typeof host === 'string' && host !== '', 'bad config: hostname must be a non-empty string'); if (port) { - assert(Number.isInteger(port, 10) && port > 0, - 'bad config: port must be a positive integer'); + assert(Number.isInteger(port, 10) && port > 0, 'bad config: port must be a positive integer'); } if (auth) { - assert(typeof auth === 'object', - 'bad config: bucket notification auth must be an object'); + assert(typeof auth === 'object', 'bad config: bucket notification auth must be an object'); } }); return bucketNotifConfig; @@ -540,21 +532,21 @@ function bucketNotifAssert(bucketNotifConfig) { function parseIntegrityChecks(config) { const integrityChecks = { - 'bucketPutACL': true, - 'bucketPutCors': true, - 'bucketPutEncryption': true, - 'bucketPutLifecycle': true, - 'bucketPutNotification': true, - 'bucketPutObjectLock': true, - 'bucketPutPolicy': true, - 'bucketPutReplication': true, - 'bucketPutVersioning': true, - 'bucketPutWebsite': true, - 'multiObjectDelete': true, - 'objectPutACL': true, - 'objectPutLegalHold': true, - 'objectPutTagging': true, - 'objectPutRetention': true, + bucketPutACL: true, + bucketPutCors: true, + bucketPutEncryption: true, + bucketPutLifecycle: true, + bucketPutNotification: true, + bucketPutObjectLock: true, + bucketPutPolicy: true, + bucketPutReplication: true, + bucketPutVersioning: true, + bucketPutWebsite: true, + multiObjectDelete: true, + objectPutACL: true, + objectPutLegalHold: true, + objectPutTagging: true, + objectPutRetention: true, }; if (config && config.integrityChecks) { @@ -596,21 +588,27 @@ function parseServerAccessLogs(config) { settings.forEach(setting => { if (setting.key in config.serverAccessLogs) { - assert(typeof config.serverAccessLogs[setting.key] === setting.type, - `bad config: serverAccessLogs.${setting.key} is not a ${setting.type}`); + assert( + typeof config.serverAccessLogs[setting.key] === setting.type, + `bad config: serverAccessLogs.${setting.key} is not a ${setting.type}`, + ); res[setting.key] = config.serverAccessLogs[setting.key]; } }); if ('mode' in config.serverAccessLogs) { - assert(validModes.includes(config.serverAccessLogs.mode), - `bad config: serverAccessLogs.mode must be one of: ${validModes.join(', ')}`); + assert( + validModes.includes(config.serverAccessLogs.mode), + `bad config: serverAccessLogs.mode must be one of: ${validModes.join(', ')}`, + ); } } if (process.env.S3_SERVER_ACCESS_LOGS_MODE) { - assert(validModes.includes(process.env.S3_SERVER_ACCESS_LOGS_MODE), - `bad config: S3_SERVER_ACCESS_LOGS_MODE must be one of: ${validModes.join(', ')}`); + assert( + validModes.includes(process.env.S3_SERVER_ACCESS_LOGS_MODE), + `bad config: S3_SERVER_ACCESS_LOGS_MODE must be one of: ${validModes.join(', ')}`, + ); res.mode = process.env.S3_SERVER_ACCESS_LOGS_MODE; } @@ -637,16 +635,13 @@ class Config extends EventEmitter { * the S3_LOCATION_FILE environment var. */ this._basePath = path.join(__dirname, '..'); - this.configPath = findConfigFile(process.env.S3_CONFIG_FILE || - 'config.json'); + this.configPath = findConfigFile(process.env.S3_CONFIG_FILE || 'config.json'); let locationConfigFileName = 'locationConfig.json'; if (process.env.CI === 'true' && !process.env.S3_END_TO_END) { - locationConfigFileName = - 'tests/locationConfig/locationConfigTests.json'; + locationConfigFileName = 'tests/locationConfig/locationConfigTests.json'; } - this.locationConfigPath = findConfigFile(process.env.S3_LOCATION_FILE || - locationConfigFileName); + this.locationConfigPath = findConfigFile(process.env.S3_LOCATION_FILE || locationConfigFileName); if (process.env.S3_REPLICATION_FILE !== undefined) { this.replicationConfigPath = process.env.S3_REPLICATION_FILE; @@ -668,13 +663,17 @@ class Config extends EventEmitter { const { providerName, region, endpoint, ak, sk, tls, noAwsArn } = config.kmsAWS; assert(providerName, 'Configuration Error: providerName must be defined in kmsAWS'); - assert(isValidProvider(providerName), - 'Configuration Error: kmsAWS.providerNamer must be lowercase alphanumeric only'); + assert( + isValidProvider(providerName), + 'Configuration Error: kmsAWS.providerNamer must be lowercase alphanumeric only', + ); assert(endpoint, 'Configuration Error: endpoint must be defined in kmsAWS'); assert(ak, 'Configuration Error: ak must be defined in kmsAWS'); assert(sk, 'Configuration Error: sk must be defined in kmsAWS'); - assert(['undefined', 'boolean'].some(type => type === typeof noAwsArn), - 'Configuration Error:: kmsAWS.noAwsArn must be a boolean or not set'); + assert( + ['undefined', 'boolean'].some(type => type === typeof noAwsArn), + 'Configuration Error:: kmsAWS.noAwsArn must be a boolean or not set', + ); kmsAWS = { providerName, @@ -700,13 +699,11 @@ class Config extends EventEmitter { // min & max TLS: One of 'TLSv1.3', 'TLSv1.2', 'TLSv1.1', or 'TLSv1' // (see https://nodejs.org/api/tls.html#tlscreatesecurecontextoptions) if (tls.minVersion !== undefined) { - assert(typeof tls.minVersion === 'string', - 'bad config: KMS AWS TLS minVersion must be a string'); + assert(typeof tls.minVersion === 'string', 'bad config: KMS AWS TLS minVersion must be a string'); kmsAWS.tls.minVersion = tls.minVersion; } if (tls.maxVersion !== undefined) { - assert(typeof tls.maxVersion === 'string', - 'bad config: KMS AWS TLS maxVersion must be a string'); + assert(typeof tls.maxVersion === 'string', 'bad config: KMS AWS TLS maxVersion must be a string'); kmsAWS.tls.maxVersion = tls.maxVersion; } if (tls.ca !== undefined) { @@ -741,11 +738,8 @@ class Config extends EventEmitter { // for customization per host host: process.env.S3KMIP_HOSTS || process.env.S3KMIP_HOST, key: this._loadTlsFile(process.env.S3KMIP_KEY || undefined), - cert: this._loadTlsFile(process.env.S3KMIP_CERT || - undefined), - ca: (process.env.S3KMIP_CA - ? process.env.S3KMIP_CA.split(',') - : []).map(ca => this._loadTlsFile(ca)), + cert: this._loadTlsFile(process.env.S3KMIP_CERT || undefined), + ca: (process.env.S3KMIP_CA ? process.env.S3KMIP_CA.split(',') : []).map(ca => this._loadTlsFile(ca)), }, }; if (transportKmip.pipelineDepth) { @@ -755,17 +749,14 @@ class Config extends EventEmitter { if (transportKmip.tls) { const { host, port, key, cert, ca } = transportKmip.tls; if (!!key !== !!cert) { - throw new Error('bad config: KMIP TLS certificate ' + - 'and key must come along'); + throw new Error('bad config: KMIP TLS certificate ' + 'and key must come along'); } if (port) { - assert(typeof port === 'number', - 'bad config: KMIP TLS Port must be a number'); + assert(typeof port === 'number', 'bad config: KMIP TLS Port must be a number'); transport.tls.port = port; } if (host) { - assert(typeof host === 'string', - 'bad config: KMIP TLS Host must be a string'); + assert(typeof host === 'string', 'bad config: KMIP TLS Host must be a string'); transport.tls.host = host; } if (key) { @@ -793,52 +784,50 @@ class Config extends EventEmitter { * time for `now' instead of client specified activation date * which also targets the present instant. */ - compoundCreateActivate: - (process.env.S3KMIP_COMPOUND_CREATE === 'true') || false, + compoundCreateActivate: process.env.S3KMIP_COMPOUND_CREATE === 'true' || false, /** Set the bucket name attribute name here if the KMIP * server supports storing custom attributes along * with the keys. */ - bucketNameAttributeName: - process.env.S3KMIP_BUCKET_ATTRIBUTE_NAME || '', + bucketNameAttributeName: process.env.S3KMIP_BUCKET_ATTRIBUTE_NAME || '', }, transport: this._parseKmipTransport({}), retries: 0, }; if (config.kmip) { assert(config.kmip.providerName, 'config.kmip.providerName must be defined'); - assert(isValidProvider(config.kmip.providerName), - 'config.kmip.providerName must be lowercase alphanumeric only'); + assert( + isValidProvider(config.kmip.providerName), + 'config.kmip.providerName must be lowercase alphanumeric only', + ); this.kmip.providerName = config.kmip.providerName; if (config.kmip.client) { if (config.kmip.client.compoundCreateActivate) { - assert(typeof config.kmip.client.compoundCreateActivate === - 'boolean'); - this.kmip.client.compoundCreateActivate = - config.kmip.client.compoundCreateActivate; + assert(typeof config.kmip.client.compoundCreateActivate === 'boolean'); + this.kmip.client.compoundCreateActivate = config.kmip.client.compoundCreateActivate; } if (config.kmip.client.bucketNameAttributeName) { - assert(typeof config.kmip.client.bucketNameAttributeName === - 'string'); - this.kmip.client.bucketNameAttributeName = - config.kmip.client.bucketNameAttributeName; + assert(typeof config.kmip.client.bucketNameAttributeName === 'string'); + this.kmip.client.bucketNameAttributeName = config.kmip.client.bucketNameAttributeName; } } if (config.kmip.transport) { if (Array.isArray(config.kmip.transport)) { - this.kmip.transport = config.kmip.transport.map(t => - this._parseKmipTransport(t)); + this.kmip.transport = config.kmip.transport.map(t => this._parseKmipTransport(t)); if (config.kmip.retries) { - assert(typeof config.kmip.retries === 'number', - 'bad config: KMIP Cluster retries must be a number'); - assert(config.kmip.retries <= this.kmip.transport.length - 1, - 'bad config: KMIP Cluster retries must be lower or equal to the number of hosts - 1'); + assert( + typeof config.kmip.retries === 'number', + 'bad config: KMIP Cluster retries must be a number', + ); + assert( + config.kmip.retries <= this.kmip.transport.length - 1, + 'bad config: KMIP Cluster retries must be lower or equal to the number of hosts - 1', + ); } else { this.kmip.retries = this.kmip.transport.length - 1; } } else { - this.kmip.transport = - this._parseKmipTransport(config.kmip.transport); + this.kmip.transport = this._parseKmipTransport(config.kmip.transport); } } } @@ -847,8 +836,7 @@ class Config extends EventEmitter { _getLocationConfig() { let locationConfig; try { - const data = fs.readFileSync(this.locationConfigPath, - { encoding: 'utf-8' }); + const data = fs.readFileSync(this.locationConfigPath, { encoding: 'utf-8' }); locationConfig = JSON.parse(data); } catch (err) { throw new Error(`could not parse location config file: @@ -861,12 +849,12 @@ class Config extends EventEmitter { Object.keys(locationConfig).forEach(l => { const details = this.locationConstraints[l].details; if (locationConfig[l].details.connector !== undefined) { - assert(typeof locationConfig[l].details.connector === - 'object', 'bad config: connector must be an object'); - if (locationConfig[l].details.connector.sproxyd !== - undefined) { - details.connector.sproxyd = parseSproxydConfig( - locationConfig[l].details.connector.sproxyd); + assert( + typeof locationConfig[l].details.connector === 'object', + 'bad config: connector must be an object', + ); + if (locationConfig[l].details.connector.sproxyd !== undefined) { + details.connector.sproxyd = parseSproxydConfig(locationConfig[l].details.connector.sproxyd); } } }); @@ -877,18 +865,14 @@ class Config extends EventEmitter { return undefined; } if (typeof tlsFileName !== 'string') { - throw new Error( - 'bad config: TLS file specification must be a string'); + throw new Error('bad config: TLS file specification must be a string'); } - const tlsFilePath = (tlsFileName[0] === '/') - ? tlsFileName - : path.join(this._basePath, tlsFileName); + const tlsFilePath = tlsFileName[0] === '/' ? tlsFileName : path.join(this._basePath, tlsFileName); let tlsFileContent; try { tlsFileContent = fs.readFileSync(tlsFilePath); } catch (err) { - throw new Error(`Could not load tls file '${tlsFileName}':` + - ` ${err.message}`); + throw new Error(`Could not load tls file '${tlsFileName}':` + ` ${err.message}`); } return tlsFileContent; } @@ -915,20 +899,18 @@ class Config extends EventEmitter { _parseEndpoints(listenOn, fieldName) { let result = []; if (listenOn !== undefined) { - assert(Array.isArray(listenOn) - && listenOn.every(e => typeof e === 'string'), - `bad config: ${fieldName} must be a list of strings`); + assert( + Array.isArray(listenOn) && listenOn.every(e => typeof e === 'string'), + `bad config: ${fieldName} must be a list of strings`, + ); result = listenOn.map(item => { const lastColon = item.lastIndexOf(':'); // if address is IPv6 format, it includes brackets // that have to be removed from the final IP address - const ipAddress = item.indexOf(']') > 0 ? - item.substr(1, lastColon - 2) : - item.substr(0, lastColon); + const ipAddress = item.indexOf(']') > 0 ? item.substr(1, lastColon - 2) : item.substr(0, lastColon); // the port should not include the colon const port = item.substr(lastColon + 1); - assert(Number.parseInt(port, 10), - `bad config: ${fieldName} port must be a positive integer`); + assert(Number.parseInt(port, 10), `bad config: ${fieldName} port must be a positive integer`); return { ip: ipAddress, port }; }); } @@ -938,32 +920,30 @@ class Config extends EventEmitter { _getConfig() { let config; try { - const data = fs.readFileSync(this.configPath, - { encoding: 'utf-8' }); + const data = fs.readFileSync(this.configPath, { encoding: 'utf-8' }); config = JSON.parse(data); } catch (err) { throw new Error(`could not parse config file: ${err.message}`); } if (this.replicationConfigPath) { try { - const repData = fs.readFileSync(this.replicationConfigPath, - { encoding: 'utf-8' }); + const repData = fs.readFileSync(this.replicationConfigPath, { encoding: 'utf-8' }); const replicationEndpoints = JSON.parse(repData); config.replicationEndpoints.push(...replicationEndpoints); } catch (err) { - throw new Error( - `could not parse replication file: ${err.message}`); + throw new Error(`could not parse replication file: ${err.message}`); } } if (config.port !== undefined) { - assert(Number.isInteger(config.port) && config.port > 0, - 'bad config: port must be a positive integer'); + assert(Number.isInteger(config.port) && config.port > 0, 'bad config: port must be a positive integer'); } if (config.internalPort !== undefined) { - assert(Number.isInteger(config.internalPort) && config.internalPort > 0, - 'bad config: internalPort must be a positive integer'); + assert( + Number.isInteger(config.internalPort) && config.internalPort > 0, + 'bad config: internalPort must be a positive integer', + ); } this.serverHeader = config.serverHeader || 'S3 Server'; @@ -981,16 +961,17 @@ class Config extends EventEmitter { this.metricsPort = 8002; if (config.metricsPort !== undefined) { - assert(Number.isInteger(config.metricsPort) && config.metricsPort > 0, - 'bad config: metricsPort must be a positive integer'); + assert( + Number.isInteger(config.metricsPort) && config.metricsPort > 0, + 'bad config: metricsPort must be a positive integer', + ); this.metricsPort = config.metricsPort; } this.metricsListenOn = this._parseEndpoints(config.metricsListenOn, 'metricsListenOn'); if (config.replicationGroupId) { - assert(typeof config.replicationGroupId === 'string', - 'bad config: replicationGroupId must be a string'); + assert(typeof config.replicationGroupId === 'string', 'bad config: replicationGroupId must be a string'); this.replicationGroupId = config.replicationGroupId; } else { this.replicationGroupId = 'RG001'; @@ -998,12 +979,10 @@ class Config extends EventEmitter { const instanceId = process.env.CLOUDSERVER_INSTANCE_ID || config.instanceId; if (instanceId) { - assert(typeof instanceId === 'string', - 'bad config: instanceId must be a string'); + assert(typeof instanceId === 'string', 'bad config: instanceId must be a string'); // versionID generation code will truncate instanceId to 6 characters // so we enforce this limit here to make the behavior predictable - assert(instanceId.length <= 6, - 'bad config: instanceId must be at most 6 characters long'); + assert(instanceId.length <= 6, 'bad config: instanceId must be at most 6 characters long'); this.instanceId = instanceId; } else { this.instanceId = uuidv4().replace(/-/g, '').slice(0, 6); @@ -1012,38 +991,59 @@ class Config extends EventEmitter { this.replicationEndpoints = []; if (config.replicationEndpoints) { const { replicationEndpoints } = config; - assert(replicationEndpoints instanceof Array, 'bad config: ' + - '`replicationEndpoints` property must be an array'); + assert( + replicationEndpoints instanceof Array, + 'bad config: ' + '`replicationEndpoints` property must be an array', + ); replicationEndpoints.forEach(replicationEndpoint => { - assert.strictEqual(typeof replicationEndpoint, 'object', - 'bad config: `replicationEndpoints` property must be an ' + - 'array of objects'); + assert.strictEqual( + typeof replicationEndpoint, + 'object', + 'bad config: `replicationEndpoints` property must be an ' + 'array of objects', + ); const { site, servers, type } = replicationEndpoint; - assert.notStrictEqual(site, undefined, 'bad config: each ' + - 'object of `replicationEndpoints` array must have a ' + - '`site` property'); - assert.strictEqual(typeof site, 'string', 'bad config: ' + - '`site` property of object in `replicationEndpoints` ' + - 'must be a string'); - assert.notStrictEqual(site, '', 'bad config: `site` property ' + - "of object in `replicationEndpoints` must not be ''"); + assert.notStrictEqual( + site, + undefined, + 'bad config: each ' + 'object of `replicationEndpoints` array must have a ' + '`site` property', + ); + assert.strictEqual( + typeof site, + 'string', + 'bad config: ' + '`site` property of object in `replicationEndpoints` ' + 'must be a string', + ); + assert.notStrictEqual( + site, + '', + 'bad config: `site` property ' + "of object in `replicationEndpoints` must not be ''", + ); if (type !== undefined) { - assert(validExternalBackends[type], 'bad config: `type` ' + - 'property of `replicationEndpoints` object must be ' + - 'a valid external backend (one of: "' + - `${Object.keys(validExternalBackends).join('", "')}")`); + assert( + validExternalBackends[type], + 'bad config: `type` ' + + 'property of `replicationEndpoints` object must be ' + + 'a valid external backend (one of: "' + + `${Object.keys(validExternalBackends).join('", "')}")`, + ); } else { - assert.notStrictEqual(servers, undefined, 'bad config: ' + - 'each object of `replicationEndpoints` array that is ' + - 'not an external backend must have `servers` property'); - assert(servers instanceof Array, 'bad config: ' + - '`servers` property of object in ' + - '`replicationEndpoints` must be an array'); + assert.notStrictEqual( + servers, + undefined, + 'bad config: ' + + 'each object of `replicationEndpoints` array that is ' + + 'not an external backend must have `servers` property', + ); + assert( + servers instanceof Array, + 'bad config: ' + '`servers` property of object in ' + '`replicationEndpoints` must be an array', + ); servers.forEach(item => { - assert(typeof item === 'string' && item !== '', + assert( + typeof item === 'string' && item !== '', 'bad config: each item of ' + - '`replicationEndpoints:servers` must be a ' + - 'non-empty string'); + '`replicationEndpoints:servers` must be a ' + + 'non-empty string', + ); }); } }); @@ -1052,27 +1052,31 @@ class Config extends EventEmitter { if (config.backbeat) { const { backbeat } = config; - assert.strictEqual(typeof backbeat.host, 'string', - 'bad config: backbeat host must be a string'); - assert(Number.isInteger(backbeat.port) && backbeat.port > 0, - 'bad config: backbeat port must be a positive integer'); + assert.strictEqual(typeof backbeat.host, 'string', 'bad config: backbeat host must be a string'); + assert( + Number.isInteger(backbeat.port) && backbeat.port > 0, + 'bad config: backbeat port must be a positive integer', + ); this.backbeat = backbeat; } if (config.workflowEngineOperator) { const { workflowEngineOperator } = config; - assert.strictEqual(typeof workflowEngineOperator.host, 'string', - 'bad config: workflowEngineOperator host must be a string'); - assert(Number.isInteger(workflowEngineOperator.port) && - workflowEngineOperator.port > 0, - 'bad config: workflowEngineOperator port not a positive integer'); + assert.strictEqual( + typeof workflowEngineOperator.host, + 'string', + 'bad config: workflowEngineOperator host must be a string', + ); + assert( + Number.isInteger(workflowEngineOperator.port) && workflowEngineOperator.port > 0, + 'bad config: workflowEngineOperator port not a positive integer', + ); this.workflowEngineOperator = workflowEngineOperator; } // legacy if (config.regions !== undefined) { - throw new Error('bad config: regions key is deprecated. ' + - 'Please use restEndpoints and locationConfig'); + throw new Error('bad config: regions key is deprecated. ' + 'Please use restEndpoints and locationConfig'); } if (config.restEndpoints !== undefined) { @@ -1087,16 +1091,19 @@ class Config extends EventEmitter { this.websiteEndpoints = []; if (config.websiteEndpoints !== undefined) { - assert(Array.isArray(config.websiteEndpoints) - && config.websiteEndpoints.every(e => typeof e === 'string'), - 'bad config: websiteEndpoints must be a list of strings'); + assert( + Array.isArray(config.websiteEndpoints) && config.websiteEndpoints.every(e => typeof e === 'string'), + 'bad config: websiteEndpoints must be a list of strings', + ); this.websiteEndpoints = config.websiteEndpoints; } this.clusters = false; if (config.clusters !== undefined) { - assert(Number.isInteger(config.clusters) && config.clusters > 0, - 'bad config: clusters must be a positive integer'); + assert( + Number.isInteger(config.clusters) && config.clusters > 0, + 'bad config: clusters must be a positive integer', + ); this.clusters = config.clusters; } if (process.env.S3BACKEND === 'mem') { @@ -1105,40 +1112,36 @@ class Config extends EventEmitter { this.isCluster = this.clusters > 1; if (config.usEastBehavior !== undefined) { - throw new Error('bad config: usEastBehavior key is deprecated. ' + - 'Please use restEndpoints and locationConfig'); + throw new Error( + 'bad config: usEastBehavior key is deprecated. ' + 'Please use restEndpoints and locationConfig', + ); } // legacy if (config.sproxyd !== undefined) { - throw new Error('bad config: sproxyd key is deprecated. ' + - 'Please use restEndpoints and locationConfig'); + throw new Error('bad config: sproxyd key is deprecated. ' + 'Please use restEndpoints and locationConfig'); } this.cdmi = {}; if (config.cdmi !== undefined) { if (config.cdmi.host !== undefined) { - assert.strictEqual(typeof config.cdmi.host, 'string', - 'bad config: cdmi host must be a string'); + assert.strictEqual(typeof config.cdmi.host, 'string', 'bad config: cdmi host must be a string'); this.cdmi.host = config.cdmi.host; } if (config.cdmi.port !== undefined) { - assert(Number.isInteger(config.cdmi.port) - && config.cdmi.port > 0, - 'bad config: cdmi port must be a positive integer'); + assert( + Number.isInteger(config.cdmi.port) && config.cdmi.port > 0, + 'bad config: cdmi port must be a positive integer', + ); this.cdmi.port = config.cdmi.port; } if (config.cdmi.path !== undefined) { - assert(typeof config.cdmi.path === 'string', - 'bad config: cdmi.path must be a string'); - assert(config.cdmi.path.length > 0, - 'bad config: cdmi.path is empty'); - assert(config.cdmi.path.charAt(0) === '/', - 'bad config: cdmi.path should start with a "/"'); + assert(typeof config.cdmi.path === 'string', 'bad config: cdmi.path must be a string'); + assert(config.cdmi.path.length > 0, 'bad config: cdmi.path is empty'); + assert(config.cdmi.path.charAt(0) === '/', 'bad config: cdmi.path should start with a "/"'); this.cdmi.path = config.cdmi.path; } if (config.cdmi.readonly !== undefined) { - assert(typeof config.cdmi.readonly === 'boolean', - 'bad config: cdmi.readonly must be a boolean'); + assert(typeof config.cdmi.readonly === 'boolean', 'bad config: cdmi.readonly must be a boolean'); this.cdmi.readonly = config.cdmi.readonly; } else { this.cdmi.readonly = true; @@ -1146,88 +1149,98 @@ class Config extends EventEmitter { } this.bucketd = { bootstrap: [] }; - if (config.bucketd !== undefined - && config.bucketd.bootstrap !== undefined) { - assert(config.bucketd.bootstrap instanceof Array - && config.bucketd.bootstrap.every( - e => typeof e === 'string'), - 'bad config: bucketd.bootstrap must be a list of strings'); + if (config.bucketd !== undefined && config.bucketd.bootstrap !== undefined) { + assert( + config.bucketd.bootstrap instanceof Array && config.bucketd.bootstrap.every(e => typeof e === 'string'), + 'bad config: bucketd.bootstrap must be a list of strings', + ); this.bucketd.bootstrap = config.bucketd.bootstrap; } this.vaultd = {}; if (config.vaultd) { if (config.vaultd.port !== undefined) { - assert(Number.isInteger(config.vaultd.port) - && config.vaultd.port > 0, - 'bad config: vaultd port must be a positive integer'); + assert( + Number.isInteger(config.vaultd.port) && config.vaultd.port > 0, + 'bad config: vaultd port must be a positive integer', + ); this.vaultd.port = config.vaultd.port; } if (config.vaultd.host !== undefined) { - assert.strictEqual(typeof config.vaultd.host, 'string', - 'bad config: vaultd host must be a string'); + assert.strictEqual(typeof config.vaultd.host, 'string', 'bad config: vaultd host must be a string'); this.vaultd.host = config.vaultd.host; } if (process.env.VAULTD_HOST !== undefined) { - assert.strictEqual(typeof process.env.VAULTD_HOST, 'string', - 'bad config: vaultd host must be a string'); + assert.strictEqual( + typeof process.env.VAULTD_HOST, + 'string', + 'bad config: vaultd host must be a string', + ); this.vaultd.host = process.env.VAULTD_HOST; } } if (config.dataClient) { this.dataClient = {}; - assert.strictEqual(typeof config.dataClient.host, 'string', - 'bad config: data client host must be ' + - 'a string'); + assert.strictEqual( + typeof config.dataClient.host, + 'string', + 'bad config: data client host must be ' + 'a string', + ); this.dataClient.host = config.dataClient.host; - assert(Number.isInteger(config.dataClient.port) - && config.dataClient.port > 0, - 'bad config: dataClient port must be a positive ' + - 'integer'); + assert( + Number.isInteger(config.dataClient.port) && config.dataClient.port > 0, + 'bad config: dataClient port must be a positive ' + 'integer', + ); this.dataClient.port = config.dataClient.port; } if (config.metadataClient) { this.metadataClient = {}; assert.strictEqual( - typeof config.metadataClient.host, 'string', - 'bad config: metadata client host must be a string'); + typeof config.metadataClient.host, + 'string', + 'bad config: metadata client host must be a string', + ); this.metadataClient.host = config.metadataClient.host; - assert(Number.isInteger(config.metadataClient.port) - && config.metadataClient.port > 0, - 'bad config: metadata client port must be a ' + - 'positive integer'); + assert( + Number.isInteger(config.metadataClient.port) && config.metadataClient.port > 0, + 'bad config: metadata client port must be a ' + 'positive integer', + ); this.metadataClient.port = config.metadataClient.port; } if (config.pfsClient) { this.pfsClient = {}; - assert.strictEqual(typeof config.pfsClient.host, 'string', - 'bad config: pfsClient host must be ' + - 'a string'); + assert.strictEqual( + typeof config.pfsClient.host, + 'string', + 'bad config: pfsClient host must be ' + 'a string', + ); this.pfsClient.host = config.pfsClient.host; - assert(Number.isInteger(config.pfsClient.port) && - config.pfsClient.port > 0, - 'bad config: pfsClient port must be a positive ' + - 'integer'); + assert( + Number.isInteger(config.pfsClient.port) && config.pfsClient.port > 0, + 'bad config: pfsClient port must be a positive ' + 'integer', + ); this.pfsClient.port = config.pfsClient.port; } if (config.dataDaemon) { this.dataDaemon = {}; assert.strictEqual( - typeof config.dataDaemon.bindAddress, 'string', - 'bad config: data daemon bind address must be a string'); + typeof config.dataDaemon.bindAddress, + 'string', + 'bad config: data daemon bind address must be a string', + ); this.dataDaemon.bindAddress = config.dataDaemon.bindAddress; - assert(Number.isInteger(config.dataDaemon.port) - && config.dataDaemon.port > 0, - 'bad config: data daemon port must be a positive ' + - 'integer'); + assert( + Number.isInteger(config.dataDaemon.port) && config.dataDaemon.port > 0, + 'bad config: data daemon port must be a positive ' + 'integer', + ); this.dataDaemon.port = config.dataDaemon.port; /** @@ -1235,9 +1248,7 @@ class Config extends EventEmitter { * backend. If no path provided, uses data at the root of * the S3 project directory. */ - this.dataDaemon.dataPath = - process.env.S3DATAPATH ? - process.env.S3DATAPATH : `${__dirname}/../localData`; + this.dataDaemon.dataPath = process.env.S3DATAPATH ? process.env.S3DATAPATH : `${__dirname}/../localData`; this.dataDaemon.noSync = process.env.S3DATA_NOSYNC === 'true'; this.dataDaemon.noCache = process.env.S3DATA_NOCACHE === 'true'; } @@ -1245,35 +1256,37 @@ class Config extends EventEmitter { if (config.pfsDaemon) { this.pfsDaemon = {}; assert.strictEqual( - typeof config.pfsDaemon.bindAddress, 'string', - 'bad config: data daemon bind address must be a string'); + typeof config.pfsDaemon.bindAddress, + 'string', + 'bad config: data daemon bind address must be a string', + ); this.pfsDaemon.bindAddress = config.pfsDaemon.bindAddress; - assert(Number.isInteger(config.pfsDaemon.port) - && config.pfsDaemon.port > 0, - 'bad config: data daemon port must be a positive ' + - 'integer'); + assert( + Number.isInteger(config.pfsDaemon.port) && config.pfsDaemon.port > 0, + 'bad config: data daemon port must be a positive ' + 'integer', + ); this.pfsDaemon.port = config.pfsDaemon.port; - this.pfsDaemon.dataPath = - process.env.PFSD_MOUNT_PATH ? - process.env.PFSD_MOUNT_PATH : `${__dirname}/../localPfs`; + this.pfsDaemon.dataPath = process.env.PFSD_MOUNT_PATH + ? process.env.PFSD_MOUNT_PATH + : `${__dirname}/../localPfs`; this.pfsDaemon.noSync = process.env.PFSD_NOSYNC === 'true'; this.pfsDaemon.noCache = process.env.PFSD_NOCACHE === 'true'; - this.pfsDaemon.isReadOnly = - process.env.PFSD_READONLY === 'true'; + this.pfsDaemon.isReadOnly = process.env.PFSD_READONLY === 'true'; } if (config.metadataDaemon) { this.metadataDaemon = {}; assert.strictEqual( - typeof config.metadataDaemon.bindAddress, 'string', - 'bad config: metadata daemon bind address must be a string'); - this.metadataDaemon.bindAddress = - config.metadataDaemon.bindAddress; - - assert(Number.isInteger(config.metadataDaemon.port) - && config.metadataDaemon.port > 0, - 'bad config: metadata daemon port must be a ' + - 'positive integer'); + typeof config.metadataDaemon.bindAddress, + 'string', + 'bad config: metadata daemon bind address must be a string', + ); + this.metadataDaemon.bindAddress = config.metadataDaemon.bindAddress; + + assert( + Number.isInteger(config.metadataDaemon.port) && config.metadataDaemon.port > 0, + 'bad config: metadata daemon port must be a ' + 'positive integer', + ); this.metadataDaemon.port = config.metadataDaemon.port; /** @@ -1281,12 +1294,11 @@ class Config extends EventEmitter { * backend. If no path provided, uses data and metadata at * the root of the S3 project directory. */ - this.metadataDaemon.metadataPath = - process.env.S3METADATAPATH ? - process.env.S3METADATAPATH : `${__dirname}/../localMetadata`; + this.metadataDaemon.metadataPath = process.env.S3METADATAPATH + ? process.env.S3METADATAPATH + : `${__dirname}/../localMetadata`; - this.metadataDaemon.restEnabled = - config.metadataDaemon.restEnabled; + this.metadataDaemon.restEnabled = config.metadataDaemon.restEnabled; this.metadataDaemon.restPort = config.metadataDaemon.restPort; } @@ -1300,48 +1312,51 @@ class Config extends EventEmitter { this.localCache = defaultLocalCache; } if (config.localCache) { - assert(typeof config.localCache === 'object', - 'config: invalid local cache configuration. localCache must ' + - 'be an object'); + assert( + typeof config.localCache === 'object', + 'config: invalid local cache configuration. localCache must ' + 'be an object', + ); if (config.localCache.sentinels) { this.localCache = { sentinels: [], name: null }; - assert(typeof config.localCache.name === 'string', - 'bad config: localCache sentinel name must be a string'); + assert( + typeof config.localCache.name === 'string', + 'bad config: localCache sentinel name must be a string', + ); this.localCache.name = config.localCache.name; - assert(Array.isArray(config.localCache.sentinels) || - typeof config.localCache.sentinels === 'string', - 'bad config: localCache sentinels' + - 'must be an array or string'); + assert( + Array.isArray(config.localCache.sentinels) || typeof config.localCache.sentinels === 'string', + 'bad config: localCache sentinels' + 'must be an array or string', + ); if (typeof config.localCache.sentinels === 'string') { config.localCache.sentinels.split(',').forEach(item => { const [host, port] = item.split(':'); - this.localCache.sentinels.push({ host, - port: Number.parseInt(port, 10) }); + this.localCache.sentinels.push({ host, port: Number.parseInt(port, 10) }); }); } else if (Array.isArray(config.localCache.sentinels)) { config.localCache.sentinels.forEach(item => { const { host, port } = item; - assert(typeof host === 'string', - 'bad config: localCache' + - 'sentinel host must be a string'); - assert(typeof port === 'number', - 'bad config: localCache' + - 'sentinel port must be a number'); + assert(typeof host === 'string', 'bad config: localCache' + 'sentinel host must be a string'); + assert(typeof port === 'number', 'bad config: localCache' + 'sentinel port must be a number'); this.localCache.sentinels.push({ host, port }); }); } } else { - assert(typeof config.localCache.host === 'string', - 'config: bad host for localCache. host must be a string'); - assert(typeof config.localCache.port === 'number', - 'config: bad port for localCache. port must be a number'); + assert( + typeof config.localCache.host === 'string', + 'config: bad host for localCache. host must be a string', + ); + assert( + typeof config.localCache.port === 'number', + 'config: bad port for localCache. port must be a number', + ); if (config.localCache.password !== undefined) { - assert(typeof config.localCache.password === 'string', - 'config: vad password for localCache. password must' + - ' be a string'); + assert( + typeof config.localCache.password === 'string', + 'config: vad password for localCache. password must' + ' be a string', + ); } this.localCache = { host: config.localCache.host, @@ -1353,11 +1368,10 @@ class Config extends EventEmitter { if (config.mongodb) { this.mongodb = config.mongodb; - if (process.env.MONGODB_AUTH_USERNAME && - process.env.MONGODB_AUTH_PASSWORD) { + if (process.env.MONGODB_AUTH_USERNAME && process.env.MONGODB_AUTH_PASSWORD) { this.mongodb.authCredentials = { - username: process.env.MONGODB_AUTH_USERNAME, - password: process.env.MONGODB_AUTH_PASSWORD, + username: process.env.MONGODB_AUTH_USERNAME, + password: process.env.MONGODB_AUTH_PASSWORD, }; } } else { @@ -1367,29 +1381,29 @@ class Config extends EventEmitter { if (config.redis) { // Fail fast to make sure we detect any bad config throw new Error( - 'config.redis is not supported anymore: it should be config.utapi.redis or config.localCache' + 'config.redis is not supported anymore: it should be config.utapi.redis or config.localCache', ); } if (config.scuba) { this.scuba = {}; if (config.scuba.host) { - assert(typeof config.scuba.host === 'string', - 'bad config: scuba host must be a string'); + assert(typeof config.scuba.host === 'string', 'bad config: scuba host must be a string'); this.scuba.host = config.scuba.host; } if (config.scuba.port) { - assert(Number.isInteger(config.scuba.port) - && config.scuba.port > 0, - 'bad config: scuba port must be a positive integer'); + assert( + Number.isInteger(config.scuba.port) && config.scuba.port > 0, + 'bad config: scuba port must be a positive integer', + ); this.scuba.port = config.scuba.port; } } if (process.env.SCUBA_HOST && process.env.SCUBA_PORT) { - assert(typeof process.env.SCUBA_HOST === 'string', - 'bad config: scuba host must be a string'); - assert(Number.isInteger(Number(process.env.SCUBA_PORT)) - && Number(process.env.SCUBA_PORT) > 0, - 'bad config: scuba port must be a positive integer'); + assert(typeof process.env.SCUBA_HOST === 'string', 'bad config: scuba host must be a string'); + assert( + Number.isInteger(Number(process.env.SCUBA_PORT)) && Number(process.env.SCUBA_PORT) > 0, + 'bad config: scuba port must be a positive integer', + ); this.scuba = { host: process.env.SCUBA_HOST, port: Number(process.env.SCUBA_PORT), @@ -1398,12 +1412,10 @@ class Config extends EventEmitter { if (this.scuba) { this.quotaEnabled = true; } - const maxStaleness = Number(process.env.QUOTA_MAX_STALENESS_MS) || - config.quota?.maxStatenessMS || - 24 * 60 * 60 * 1000; + const maxStaleness = + Number(process.env.QUOTA_MAX_STALENESS_MS) || config.quota?.maxStatenessMS || 24 * 60 * 60 * 1000; assert(Number.isInteger(maxStaleness), 'bad config: maxStalenessMS must be an integer'); - const enableInflights = process.env.QUOTA_ENABLE_INFLIGHTS === 'true' || - config.quota?.enableInflights || false; + const enableInflights = process.env.QUOTA_ENABLE_INFLIGHTS === 'true' || config.quota?.enableInflights || false; this.quota = { maxStaleness, enableInflights, @@ -1411,30 +1423,29 @@ class Config extends EventEmitter { if (config.utapi) { this.utapi = { component: 's3' }; if (config.utapi.host) { - assert(typeof config.utapi.host === 'string', - 'bad config: utapi host must be a string'); + assert(typeof config.utapi.host === 'string', 'bad config: utapi host must be a string'); this.utapi.host = config.utapi.host; } if (config.utapi.port) { - assert(Number.isInteger(config.utapi.port) - && config.utapi.port > 0, - 'bad config: utapi port must be a positive integer'); + assert( + Number.isInteger(config.utapi.port) && config.utapi.port > 0, + 'bad config: utapi port must be a positive integer', + ); this.utapi.port = config.utapi.port; } if (utapiVersion === 1) { if (config.utapi.workers !== undefined) { - assert(Number.isInteger(config.utapi.workers) - && config.utapi.workers > 0, - 'bad config: utapi workers must be a positive integer'); + assert( + Number.isInteger(config.utapi.workers) && config.utapi.workers > 0, + 'bad config: utapi workers must be a positive integer', + ); this.utapi.workers = config.utapi.workers; } // Utapi uses the same localCache config defined for S3 to avoid // config duplication. - assert(config.localCache, 'missing required property of utapi ' + - 'configuration: localCache'); + assert(config.localCache, 'missing required property of utapi ' + 'configuration: localCache'); this.utapi.localCache = this.localCache; - assert(config.utapi.redis, 'missing required property of utapi ' + - 'configuration: redis'); + assert(config.utapi.redis, 'missing required property of utapi ' + 'configuration: redis'); this.utapi.redis = parseRedisConfig(config.utapi.redis); if (this.utapi.redis.retry === undefined) { this.utapi.redis.retry = { @@ -1453,29 +1464,36 @@ class Config extends EventEmitter { this.utapi.enabledOperationCounters = []; if (config.utapi.enabledOperationCounters !== undefined) { const { enabledOperationCounters } = config.utapi; - assert(Array.isArray(enabledOperationCounters), - 'bad config: utapi.enabledOperationCounters must be an ' + - 'array'); - assert(enabledOperationCounters.length > 0, - 'bad config: utapi.enabledOperationCounters cannot be ' + - 'empty'); + assert( + Array.isArray(enabledOperationCounters), + 'bad config: utapi.enabledOperationCounters must be an ' + 'array', + ); + assert( + enabledOperationCounters.length > 0, + 'bad config: utapi.enabledOperationCounters cannot be ' + 'empty', + ); this.utapi.enabledOperationCounters = enabledOperationCounters; } this.utapi.disableOperationCounters = false; if (config.utapi.disableOperationCounters !== undefined) { const { disableOperationCounters } = config.utapi; - assert(typeof disableOperationCounters === 'boolean', - 'bad config: utapi.disableOperationCounters must be a ' + - 'boolean'); + assert( + typeof disableOperationCounters === 'boolean', + 'bad config: utapi.disableOperationCounters must be a ' + 'boolean', + ); this.utapi.disableOperationCounters = disableOperationCounters; } - if (config.utapi.disableOperationCounters !== undefined && - config.utapi.enabledOperationCounters !== undefined) { - assert(config.utapi.disableOperationCounters === false, + if ( + config.utapi.disableOperationCounters !== undefined && + config.utapi.enabledOperationCounters !== undefined + ) { + assert( + config.utapi.disableOperationCounters === false, 'bad config: conflicting rules: ' + - 'utapi.disableOperationCounters and ' + - 'utapi.enabledOperationCounters cannot both be ' + - 'specified'); + 'utapi.disableOperationCounters and ' + + 'utapi.enabledOperationCounters cannot both be ' + + 'specified', + ); } if (config.utapi.component) { this.utapi.component = config.utapi.component; @@ -1483,17 +1501,23 @@ class Config extends EventEmitter { // (optional) The value of the replay schedule should be cron-style // scheduling. For example, every five minutes: '*/5 * * * *'. if (config.utapi.replaySchedule) { - assert(typeof config.utapi.replaySchedule === 'string', 'bad' + - 'config: utapi.replaySchedule must be a string'); + assert( + typeof config.utapi.replaySchedule === 'string', + 'bad' + 'config: utapi.replaySchedule must be a string', + ); this.utapi.replaySchedule = config.utapi.replaySchedule; } // (optional) The number of elements processed by each call to the // Redis local cache during a replay. For example, 50. if (config.utapi.batchSize) { - assert(typeof config.utapi.batchSize === 'number', 'bad' + - 'config: utapi.batchSize must be a number'); - assert(config.utapi.batchSize > 0, 'bad config:' + - 'utapi.batchSize must be a number greater than 0'); + assert( + typeof config.utapi.batchSize === 'number', + 'bad' + 'config: utapi.batchSize must be a number', + ); + assert( + config.utapi.batchSize > 0, + 'bad config:' + 'utapi.batchSize must be a number greater than 0', + ); this.utapi.batchSize = config.utapi.batchSize; } @@ -1501,16 +1525,20 @@ class Config extends EventEmitter { // Disabled by default this.utapi.expireMetrics = false; if (config.utapi.expireMetrics !== undefined) { - assert(typeof config.utapi.expireMetrics === 'boolean', 'bad' + - 'config: utapi.expireMetrics must be a boolean'); + assert( + typeof config.utapi.expireMetrics === 'boolean', + 'bad' + 'config: utapi.expireMetrics must be a boolean', + ); this.utapi.expireMetrics = config.utapi.expireMetrics; } // (optional) TTL controlling the expiry for bucket level metrics // keys when expireMetrics is enabled this.utapi.expireMetricsTTL = 0; if (config.utapi.expireMetricsTTL !== undefined) { - assert(typeof config.utapi.expireMetricsTTL === 'number', - 'bad config: utapi.expireMetricsTTL must be a number'); + assert( + typeof config.utapi.expireMetricsTTL === 'number', + 'bad config: utapi.expireMetricsTTL must be a number', + ); this.utapi.expireMetricsTTL = config.utapi.expireMetricsTTL; } @@ -1522,40 +1550,42 @@ class Config extends EventEmitter { if (utapiVersion === 2 && config.utapi.filter) { const { filter: filterConfig } = config.utapi; const utapiResourceFilters = {}; - allowedUtapiEventFilterFields.forEach( - field => allowedUtapiEventFilterStates.forEach( - state => { - const resources = (filterConfig[state] && filterConfig[state][field]) || null; - if (resources) { - assert.strictEqual(utapiResourceFilters[field], undefined, - `bad config: utapi.filter.${state}.${field} can't define an allow and a deny list`); - assert(resources.every(r => typeof r === 'string'), - `bad config: utapi.filter.${state}.${field} must be an array of strings`); - utapiResourceFilters[field] = { [state]: new Set(resources) }; - } + allowedUtapiEventFilterFields.forEach(field => + allowedUtapiEventFilterStates.forEach(state => { + const resources = (filterConfig[state] && filterConfig[state][field]) || null; + if (resources) { + assert.strictEqual( + utapiResourceFilters[field], + undefined, + `bad config: utapi.filter.${state}.${field} can't define an allow and a deny list`, + ); + assert( + resources.every(r => typeof r === 'string'), + `bad config: utapi.filter.${state}.${field} must be an array of strings`, + ); + utapiResourceFilters[field] = { [state]: new Set(resources) }; } - )); + }), + ); this.utapi.filter = utapiResourceFilters; } } - if (Object.keys(this.locationConstraints).some( - loc => this.locationConstraints[loc].sizeLimitGB)) { - assert(this.utapi && this.utapi.metrics && - this.utapi.metrics.includes('location'), + if (Object.keys(this.locationConstraints).some(loc => this.locationConstraints[loc].sizeLimitGB)) { + assert( + this.utapi && this.utapi.metrics && this.utapi.metrics.includes('location'), 'bad config: if storage size limit set on a location ' + - 'constraint, Utapi must also be configured correctly'); + 'constraint, Utapi must also be configured correctly', + ); } this.log = { logLevel: 'debug', dumpLevel: 'error' }; if (config.log !== undefined) { if (config.log.logLevel !== undefined) { - assert(typeof config.log.logLevel === 'string', - 'bad config: log.logLevel must be a string'); + assert(typeof config.log.logLevel === 'string', 'bad config: log.logLevel must be a string'); this.log.logLevel = config.log.logLevel; } if (config.log.dumpLevel !== undefined) { - assert(typeof config.log.dumpLevel === 'string', - 'bad config: log.dumpLevel must be a string'); + assert(typeof config.log.dumpLevel === 'string', 'bad config: log.dumpLevel must be a string'); this.log.dumpLevel = config.log.dumpLevel; } } @@ -1563,8 +1593,10 @@ class Config extends EventEmitter { this.kms = {}; if (config.kms) { assert(config.kms.providerName, 'config.kms.providerName must be provided'); - assert(isValidProvider(config.kms.providerName), - 'config.kms.providerName must be lowercase alphanumeric only'); + assert( + isValidProvider(config.kms.providerName), + 'config.kms.providerName must be lowercase alphanumeric only', + ); assert(typeof config.kms.userName === 'string'); assert(typeof config.kms.password === 'string'); this.kms.providerName = config.kms.providerName; @@ -1589,13 +1621,14 @@ class Config extends EventEmitter { const globalEncryptionEnabled = config.globalEncryptionEnabled; this.globalEncryptionEnabled = globalEncryptionEnabled || false; - assert(typeof this.globalEncryptionEnabled === 'boolean', - 'config.globalEncryptionEnabled must be a boolean'); + assert(typeof this.globalEncryptionEnabled === 'boolean', 'config.globalEncryptionEnabled must be a boolean'); const defaultEncryptionKeyPerAccount = config.defaultEncryptionKeyPerAccount; this.defaultEncryptionKeyPerAccount = defaultEncryptionKeyPerAccount || false; - assert(typeof this.defaultEncryptionKeyPerAccount === 'boolean', - 'config.defaultEncryptionKeyPerAccount must be a boolean'); + assert( + typeof this.defaultEncryptionKeyPerAccount === 'boolean', + 'config.defaultEncryptionKeyPerAccount must be a boolean', + ); this.kmsHideScalityArn = Object.hasOwnProperty.call(config, 'kmsHideScalityArn') ? config.kmsHideScalityArn @@ -1604,16 +1637,17 @@ class Config extends EventEmitter { this.healthChecks = defaultHealthChecks; if (config.healthChecks && config.healthChecks.allowFrom) { - assert(config.healthChecks.allowFrom instanceof Array, - 'config: invalid healthcheck configuration. allowFrom must ' + - 'be an array'); + assert( + config.healthChecks.allowFrom instanceof Array, + 'config: invalid healthcheck configuration. allowFrom must ' + 'be an array', + ); config.healthChecks.allowFrom.forEach(item => { - assert(typeof item === 'string', - 'config: invalid healthcheck configuration. allowFrom IP ' + - 'address must be a string'); + assert( + typeof item === 'string', + 'config: invalid healthcheck configuration. allowFrom IP ' + 'address must be a string', + ); }); - this.healthChecks.allowFrom = defaultHealthChecks.allowFrom - .concat(config.healthChecks.allowFrom); + this.healthChecks.allowFrom = defaultHealthChecks.allowFrom.concat(config.healthChecks.allowFrom); } /** * CLDSRV-740: S3C with nginx s3-frontend needs the healthcheck on the @@ -1622,23 +1656,21 @@ class Config extends EventEmitter { this.healthChecks.enableInternalRoute = config.healthChecks?.enableInternalRoute || false; if (config.certFilePaths) { - assert(typeof config.certFilePaths === 'object' && - typeof config.certFilePaths.key === 'string' && - typeof config.certFilePaths.cert === 'string' && (( - config.certFilePaths.ca && - typeof config.certFilePaths.ca === 'string') || - !config.certFilePaths.ca) - ); + assert( + typeof config.certFilePaths === 'object' && + typeof config.certFilePaths.key === 'string' && + typeof config.certFilePaths.cert === 'string' && + ((config.certFilePaths.ca && typeof config.certFilePaths.ca === 'string') || + !config.certFilePaths.ca), + ); } - const { key, cert, ca } = config.certFilePaths ? - config.certFilePaths : {}; + const { key, cert, ca } = config.certFilePaths ? config.certFilePaths : {}; let certObj = undefined; if (key && cert) { certObj = assertCertPaths(key, cert, ca, this._basePath); } else if (key || cert) { - throw new Error('bad config: both certFilePaths.key and ' + - 'certFilePaths.cert must be defined'); + throw new Error('bad config: both certFilePaths.key and ' + 'certFilePaths.cert must be defined'); } if (certObj) { if (Object.keys(certObj.certs).length > 0) { @@ -1650,30 +1682,29 @@ class Config extends EventEmitter { } this.outboundProxy = {}; - const envProxy = process.env.HTTP_PROXY || process.env.HTTPS_PROXY - || process.env.http_proxy || process.env.https_proxy; + const envProxy = + process.env.HTTP_PROXY || process.env.HTTPS_PROXY || process.env.http_proxy || process.env.https_proxy; const p = config.outboundProxy; const proxyUrl = envProxy || (p ? p.url : ''); if (proxyUrl) { - assert(typeof proxyUrl === 'string', - 'bad proxy config: url must be a string'); + assert(typeof proxyUrl === 'string', 'bad proxy config: url must be a string'); const { protocol, hostname, port, auth } = url.parse(proxyUrl); - assert(protocol === 'http:' || protocol === 'https:', - 'bad proxy config: protocol must be http or https'); - assert(typeof hostname === 'string' && hostname !== '', - 'bad proxy config: hostname must be a non-empty string'); + assert(protocol === 'http:' || protocol === 'https:', 'bad proxy config: protocol must be http or https'); + assert( + typeof hostname === 'string' && hostname !== '', + 'bad proxy config: hostname must be a non-empty string', + ); if (port) { const portInt = Number.parseInt(port, 10); - assert(!Number.isNaN(portInt) && portInt > 0, - 'bad proxy config: port must be a number greater than 0'); + assert(!Number.isNaN(portInt) && portInt > 0, 'bad proxy config: port must be a number greater than 0'); } if (auth) { - assert(typeof auth === 'string', - 'bad proxy config: auth must be string'); + assert(typeof auth === 'string', 'bad proxy config: auth must be string'); const authArray = auth.split(':'); - assert(authArray.length === 2 && authArray[0].length > 0 - && authArray[1].length > 0, 'bad proxy config: ' + - 'auth must be of format username:password'); + assert( + authArray.length === 2 && authArray[0].length > 0 && authArray[1].length > 0, + 'bad proxy config: ' + 'auth must be of format username:password', + ); } this.outboundProxy.url = proxyUrl; this.outboundProxy.certs = {}; @@ -1682,23 +1713,18 @@ class Config extends EventEmitter { const cert = p ? p.cert : ''; const caBundle = envCert || (p ? p.caBundle : ''); if (p) { - assert(typeof p === 'object', - 'bad config: "proxy" should be an object'); + assert(typeof p === 'object', 'bad config: "proxy" should be an object'); } if (key) { - assert(typeof key === 'string', - 'bad config: proxy.key should be a string'); + assert(typeof key === 'string', 'bad config: proxy.key should be a string'); } if (cert) { - assert(typeof cert === 'string', - 'bad config: proxy.cert should be a string'); + assert(typeof cert === 'string', 'bad config: proxy.cert should be a string'); } if (caBundle) { - assert(typeof caBundle === 'string', - 'bad config: proxy.caBundle should be a string'); + assert(typeof caBundle === 'string', 'bad config: proxy.caBundle should be a string'); } - const certObj = - assertCertPaths(key, cert, caBundle, this._basePath); + const certObj = assertCertPaths(key, cert, caBundle, this._basePath); this.outboundProxy.certs = certObj.certs; } @@ -1707,16 +1733,18 @@ class Config extends EventEmitter { this.managementAgent.host = 'localhost'; if (config.managementAgent !== undefined) { if (config.managementAgent.port !== undefined) { - assert(Number.isInteger(config.managementAgent.port) - && config.managementAgent.port > 0, - 'bad config: managementAgent port must be a positive ' + - 'integer'); + assert( + Number.isInteger(config.managementAgent.port) && config.managementAgent.port > 0, + 'bad config: managementAgent port must be a positive ' + 'integer', + ); this.managementAgent.port = config.managementAgent.port; } if (config.managementAgent.host !== undefined) { - assert.strictEqual(typeof config.managementAgent.host, 'string', - 'bad config: management agent host must ' + - 'be a string'); + assert.strictEqual( + typeof config.managementAgent.host, + 'string', + 'bad config: management agent host must ' + 'be a string', + ); this.managementAgent.host = config.managementAgent.host; } } @@ -1724,10 +1752,7 @@ class Config extends EventEmitter { // Ephemeral token to protect the reporting endpoint: // try inherited from parent first, then hardcoded in conf file, // then create a fresh one as last resort. - this.reportToken = - process.env.REPORT_TOKEN || - config.reportToken || - uuidv4(); + this.reportToken = process.env.REPORT_TOKEN || config.reportToken || uuidv4(); // External backends // Currently supports configuring httpAgent(s) for keepAlive @@ -1736,29 +1761,28 @@ class Config extends EventEmitter { const extBackendsConfig = Object.keys(config.externalBackends); extBackendsConfig.forEach(b => { // assert that it's a valid backend - assert(validExternalBackends[b] !== undefined, + assert( + validExternalBackends[b] !== undefined, `bad config: ${b} is not one of valid external backends: ` + - `${Object.keys(validExternalBackends).join(', ')}`); + `${Object.keys(validExternalBackends).join(', ')}`, + ); const { httpAgent } = config.externalBackends[b]; - assert(typeof httpAgent === 'object', - `bad config: ${b} must have httpAgent object defined`); - const { keepAlive, keepAliveMsecs, maxFreeSockets, maxSockets } - = httpAgent; - assert(typeof keepAlive === 'boolean', - `bad config: ${b}.httpAgent.keepAlive must be a boolean`); - assert(typeof keepAliveMsecs === 'number' && - httpAgent.keepAliveMsecs > 0, - `bad config: ${b}.httpAgent.keepAliveMsecs must be` + - ' a number > 0'); - assert(typeof maxFreeSockets === 'number' && - httpAgent.maxFreeSockets >= 0, - `bad config: ${b}.httpAgent.maxFreeSockets must be ` + - 'a number >= 0'); - assert((typeof maxSockets === 'number' && maxSockets >= 0) || - maxSockets === null, - `bad config: ${b}.httpAgent.maxFreeSockets must be ` + - 'null or a number >= 0'); + assert(typeof httpAgent === 'object', `bad config: ${b} must have httpAgent object defined`); + const { keepAlive, keepAliveMsecs, maxFreeSockets, maxSockets } = httpAgent; + assert(typeof keepAlive === 'boolean', `bad config: ${b}.httpAgent.keepAlive must be a boolean`); + assert( + typeof keepAliveMsecs === 'number' && httpAgent.keepAliveMsecs > 0, + `bad config: ${b}.httpAgent.keepAliveMsecs must be` + ' a number > 0', + ); + assert( + typeof maxFreeSockets === 'number' && httpAgent.maxFreeSockets >= 0, + `bad config: ${b}.httpAgent.maxFreeSockets must be ` + 'a number >= 0', + ); + assert( + (typeof maxSockets === 'number' && maxSockets >= 0) || maxSockets === null, + `bad config: ${b}.httpAgent.maxFreeSockets must be ` + 'null or a number >= 0', + ); Object.assign(this.externalBackends[b].httpAgent, httpAgent); }); } @@ -1802,9 +1826,11 @@ class Config extends EventEmitter { // maxScannedLifecycleListingEntries > 2 is required as a minimum because we must // scan at least three entries to determine version eligibility. // Two entries representing the master key and the following one representing the non-current version. - assert(Number.isInteger(config.maxScannedLifecycleListingEntries) && - config.maxScannedLifecycleListingEntries > 2, - 'bad config: maxScannedLifecycleListingEntries must be greater than 2'); + assert( + Number.isInteger(config.maxScannedLifecycleListingEntries) && + config.maxScannedLifecycleListingEntries > 2, + 'bad config: maxScannedLifecycleListingEntries must be greater than 2', + ); this.maxScannedLifecycleListingEntries = config.maxScannedLifecycleListingEntries; } @@ -1814,18 +1840,23 @@ class Config extends EventEmitter { this.apiBodySizeLimits = { ...constants.defaultApiBodySizeLimits }; if (config.apiBodySizeLimits) { - assert(typeof config.apiBodySizeLimits === 'object' && - !Array.isArray(config.apiBodySizeLimits), - 'bad config: apiBodySizeLimits must be an object'); + assert( + typeof config.apiBodySizeLimits === 'object' && !Array.isArray(config.apiBodySizeLimits), + 'bad config: apiBodySizeLimits must be an object', + ); for (const [apiKey, limit] of Object.entries(config.apiBodySizeLimits)) { // Only allow modifications of predefined APIs from constants - assert(Object.hasOwn(constants.defaultApiBodySizeLimits, apiKey), + assert( + Object.hasOwn(constants.defaultApiBodySizeLimits, apiKey), `bad config: apiBodySizeLimits for "${apiKey}" cannot be configured. ` + - `Valid APIs are: ${Object.keys(constants.defaultApiBodySizeLimits).join(', ')}`); + `Valid APIs are: ${Object.keys(constants.defaultApiBodySizeLimits).join(', ')}`, + ); - assert(Number.isInteger(limit) && limit > 0, - `bad config: apiBodySizeLimits for "${apiKey}" must be a positive integer`); + assert( + Number.isInteger(limit) && limit > 0, + `bad config: apiBodySizeLimits for "${apiKey}" must be a positive integer`, + ); this.apiBodySizeLimits[apiKey] = limit; } } @@ -1835,8 +1866,8 @@ class Config extends EventEmitter { * S3C-10336: PutObject max size of 5GB is new in 9.5.1 * Provides a way to bypass the new validation if it breaks customer workflows */ - this.bypassMaxPutObjectSize = process.env.BYPASS_MAX_PUT_OBJECT_SIZE === 'true' - || config.bypassMaxPutObjectSize || false; + this.bypassMaxPutObjectSize = + process.env.BYPASS_MAX_PUT_OBJECT_SIZE === 'true' || config.bypassMaxPutObjectSize || false; /** * S3C-10370: Before 9.5.1, there was no limit on the key length. @@ -1847,8 +1878,10 @@ class Config extends EventEmitter { process.env.OVERRIDE_OBJECT_KEY_BYTE_LIMIT || config.overrideObjectKeyByteLimit; if (overrideObjectKeyByteLimit !== null && overrideObjectKeyByteLimit !== undefined) { this.objectKeyByteLimit = parseInt(overrideObjectKeyByteLimit, 10); - assert(Number.isInteger(this.objectKeyByteLimit) && this.objectKeyByteLimit >= 0, - 'bad config: overrideObjectKeyByteLimit must be a positive integer'); + assert( + Number.isInteger(this.objectKeyByteLimit) && this.objectKeyByteLimit >= 0, + 'bad config: overrideObjectKeyByteLimit must be a positive integer', + ); } this.enableVeeamRoute = true; @@ -1887,9 +1920,12 @@ class Config extends EventEmitter { // decreases the weight attributed to a day in order to expedite the lifecycle of objects. const timeProgressionFactor = Number.parseInt(process.env.TIME_PROGRESSION_FACTOR, 10) || 1; - const isIncompatible = (expireOneDayEarlier || transitionOneDayEarlier) && (timeProgressionFactor > 1); - assert(!isIncompatible, 'The environment variables "EXPIRE_ONE_DAY_EARLIER" or ' + - '"TRANSITION_ONE_DAY_EARLIER" are not compatible with the "TIME_PROGRESSION_FACTOR" variable.'); + const isIncompatible = (expireOneDayEarlier || transitionOneDayEarlier) && timeProgressionFactor > 1; + assert( + !isIncompatible, + 'The environment variables "EXPIRE_ONE_DAY_EARLIER" or ' + + '"TRANSITION_ONE_DAY_EARLIER" are not compatible with the "TIME_PROGRESSION_FACTOR" variable.', + ); // The scaledMsPerDay value is initially set to the number of milliseconds per day // (24 * 60 * 60 * 1000) as the default value. @@ -1925,9 +1961,9 @@ class Config extends EventEmitter { let quota = 'none'; if (process.env.S3BACKEND) { const validBackends = ['mem', 'file', 'scality', 'cdmi']; - assert(validBackends.indexOf(process.env.S3BACKEND) > -1, - 'bad environment variable: S3BACKEND environment variable ' + - 'should be one of mem/file/scality/cdmi' + assert( + validBackends.indexOf(process.env.S3BACKEND) > -1, + 'bad environment variable: S3BACKEND environment variable ' + 'should be one of mem/file/scality/cdmi', ); auth = process.env.S3BACKEND; data = process.env.S3BACKEND; @@ -1941,11 +1977,11 @@ class Config extends EventEmitter { // Auth only checks for 'mem' since mem === file auth = 'mem'; let authData; - if (process.env.SCALITY_ACCESS_KEY_ID && - process.env.SCALITY_SECRET_ACCESS_KEY) { + if (process.env.SCALITY_ACCESS_KEY_ID && process.env.SCALITY_SECRET_ACCESS_KEY) { authData = buildAuthDataAccount( - process.env.SCALITY_ACCESS_KEY_ID, - process.env.SCALITY_SECRET_ACCESS_KEY); + process.env.SCALITY_ACCESS_KEY_ID, + process.env.SCALITY_SECRET_ACCESS_KEY, + ); } else { authData = this._getAuthData(); } @@ -1953,7 +1989,7 @@ class Config extends EventEmitter { throw new Error('bad config: invalid auth config file.'); } this.authData = authData; - } else if (auth === 'multiple') { + } else if (auth === 'multiple') { const authData = this._getAuthData(); if (validateAuthConfig(authData)) { throw new Error('bad config: invalid auth config file.'); @@ -1963,18 +1999,18 @@ class Config extends EventEmitter { if (process.env.S3DATA) { const validData = ['mem', 'file', 'scality', 'multiple']; - assert(validData.indexOf(process.env.S3DATA) > -1, - 'bad environment variable: S3DATA environment variable ' + - 'should be one of mem/file/scality/multiple' + assert( + validData.indexOf(process.env.S3DATA) > -1, + 'bad environment variable: S3DATA environment variable ' + 'should be one of mem/file/scality/multiple', ); data = process.env.S3DATA; } if (data === 'scality' || data === 'multiple') { data = 'multiple'; } - assert(this.locationConstraints !== undefined && - this.restEndpoints !== undefined, - 'bad config: locationConstraints and restEndpoints must be set' + assert( + this.locationConstraints !== undefined && this.restEndpoints !== undefined, + 'bad config: locationConstraints and restEndpoints must be set', ); if (process.env.S3METADATA) { @@ -1997,13 +2033,12 @@ class Config extends EventEmitter { // Mongodb backend does not support null keys, so we must enforce null version compatibility // mode. With other backends (esp. metadata), this is used during migration from v0 to v1 // bucket format. - this.nullVersionCompatMode = (metadata === 'mongodb') || - (process.env.ENABLE_NULL_VERSION_COMPAT_MODE === 'true'); + this.nullVersionCompatMode = metadata === 'mongodb' || process.env.ENABLE_NULL_VERSION_COMPAT_MODE === 'true'; // Multi-object delete optimizations is only supported for MongoDB at the moment. It relies // on `getObjectsMD()` to return the objects in a single call, which is not supported by // other backends. - this.multiObjectDeleteEnableOptimizations &&= (metadata === 'mongodb'); + this.multiObjectDeleteEnableOptimizations &&= metadata === 'mongodb'; } _sseMigration(config) { @@ -2016,14 +2051,12 @@ class Config extends EventEmitter { this.sseMigration = {}; const { previousKeyType, previousKeyProtocol, previousKeyProvider } = config.sseMigration; if (!previousKeyType) { - assert.fail( - 'NotImplemented: No dynamic KMS key migration. Set sseMigration.previousKeyType'); + assert.fail('NotImplemented: No dynamic KMS key migration. Set sseMigration.previousKeyType'); } // If previousKeyType is provided it's used as static value to migrate the format of the key // without additional dynamic evaluation if the key provider is unknown. - assert(isValidType(previousKeyType), - 'ssenMigration.previousKeyType must be "internal" or "external"'); + assert(isValidType(previousKeyType), 'ssenMigration.previousKeyType must be "internal" or "external"'); this.sseMigration.previousKeyType = previousKeyType; let expectedProtocol; @@ -2034,25 +2067,28 @@ class Config extends EventEmitter { expectedProtocol = [KmsProtocol.scality, KmsProtocol.mem, KmsProtocol.file]; } else if (previousKeyType === KmsType.external) { // No defaults allowed for external provider - assert(previousKeyProtocol, - 'sseMigration.previousKeyProtocol must be defined for external provider'); + assert(previousKeyProtocol, 'sseMigration.previousKeyProtocol must be defined for external provider'); this.sseMigration.previousKeyProtocol = previousKeyProtocol; - assert(previousKeyProvider, - 'sseMigration.previousKeyProvider must be defined for external provider'); + assert(previousKeyProvider, 'sseMigration.previousKeyProvider must be defined for external provider'); this.sseMigration.previousKeyProvider = previousKeyProvider; expectedProtocol = [KmsProtocol.kmip, KmsProtocol.aws_kms]; } - assert(isValidProtocol(previousKeyType, this.sseMigration.previousKeyProtocol), - `sseMigration.previousKeyProtocol must be one of ${expectedProtocol}`); - assert(isValidProvider(previousKeyProvider), - 'sseMigration.previousKeyProvider must be lowercase alphanumeric only'); + assert( + isValidProtocol(previousKeyType, this.sseMigration.previousKeyProtocol), + `sseMigration.previousKeyProtocol must be one of ${expectedProtocol}`, + ); + assert( + isValidProvider(previousKeyProvider), + 'sseMigration.previousKeyProvider must be lowercase alphanumeric only', + ); if (this.sseMigration.previousKeyType === KmsType.external) { if ([KmsProtocol.file, KmsProtocol.mem].includes(this.backends.kms)) { assert.fail( `sseMigration.previousKeyType "external" can't migrate to "internal" KMS provider ${ - this.backends.kms}` + this.backends.kms + }`, ); } // We'd have to compare protocol & providerName @@ -2071,10 +2107,7 @@ class Config extends EventEmitter { } getGcpBucketNames(locationConstraint) { - const { - bucketName, - mpuBucketName, - } = this.locationConstraints[locationConstraint].details; + const { bucketName, mpuBucketName } = this.locationConstraints[locationConstraint].details; return { bucketName, mpuBucketName }; } @@ -2100,9 +2133,10 @@ class Config extends EventEmitter { } setReplicationEndpoints(locationConstraints) { - this.replicationEndpoints = - Object.keys(locationConstraints) - .map(key => ({ site: key, type: locationConstraints[key].type })); + this.replicationEndpoints = Object.keys(locationConstraints).map(key => ({ + site: key, + type: locationConstraints[key].type, + })); } getAzureEndpoint(locationConstraint) { @@ -2119,7 +2153,7 @@ class Config extends EventEmitter { getAzureStorageAccountName(locationConstraint) { const accountName = azureGetStorageAccountName( locationConstraint, - this.locationConstraints[locationConstraint].details + this.locationConstraints[locationConstraint].details, ); if (accountName) { return accountName; @@ -2147,31 +2181,27 @@ class Config extends EventEmitter { } getAzureStorageCredentials(locationConstraint) { - return azureGetLocationCredentials( - locationConstraint, - this.locationConstraints[locationConstraint].details - ); + return azureGetLocationCredentials(locationConstraint, this.locationConstraints[locationConstraint].details); } getPfsDaemonEndpoint(locationConstraint) { - return process.env[`${locationConstraint}_PFSD_ENDPOINT`] || - this.locationConstraints[locationConstraint].details.pfsDaemonEndpoint; + return ( + process.env[`${locationConstraint}_PFSD_ENDPOINT`] || + this.locationConstraints[locationConstraint].details.pfsDaemonEndpoint + ); } isSameAzureAccount(locationConstraintSrc, locationConstraintDest) { if (!locationConstraintDest) { return true; } - const azureSrcAccount = - this.getAzureStorageAccountName(locationConstraintSrc); - const azureDestAccount = - this.getAzureStorageAccountName(locationConstraintDest); + const azureSrcAccount = this.getAzureStorageAccountName(locationConstraintSrc); + const azureDestAccount = this.getAzureStorageAccountName(locationConstraintDest); return azureSrcAccount === azureDestAccount; } isAWSServerSideEncryption(locationConstraint) { - return this.locationConstraints[locationConstraint].details - .serverSideEncryption === true; + return this.locationConstraints[locationConstraint].details.serverSideEncryption === true; } getPublicInstanceId() { @@ -2179,9 +2209,7 @@ class Config extends EventEmitter { } setPublicInstanceId(instanceId) { - this.publicInstanceId = crypto.createHash('sha256') - .update(instanceId) - .digest('hex'); + this.publicInstanceId = crypto.createHash('sha256').update(instanceId).digest('hex'); } isQuotaEnabled() { diff --git a/lib/api/apiUtils/authorization/permissionChecks.js b/lib/api/apiUtils/authorization/permissionChecks.js index d82a1bc392..2459d552d4 100644 --- a/lib/api/apiUtils/authorization/permissionChecks.js +++ b/lib/api/apiUtils/authorization/permissionChecks.js @@ -16,8 +16,7 @@ const { } = constants; // whitelist buckets to allow public read on objects -const publicReadBuckets = process.env.ALLOW_PUBLIC_READ_BUCKETS - ? process.env.ALLOW_PUBLIC_READ_BUCKETS.split(',') : []; +const publicReadBuckets = process.env.ALLOW_PUBLIC_READ_BUCKETS ? process.env.ALLOW_PUBLIC_READ_BUCKETS.split(',') : []; function getServiceAccountProperties(canonicalID) { const canonicalIDArray = canonicalID.split('/'); @@ -65,12 +64,9 @@ const checkBucketPolicyResult = Object.freeze({ function checkBucketAcls(bucket, requestType, canonicalID, mainApiCall) { // Same logic applies on the Versioned APIs, so let's simplify it. - let requestTypeParsed = requestType.endsWith('Version') ? - requestType.slice(0, 'Version'.length * -1) : requestType; - requestTypeParsed = actionsToConsiderAsObjectPut.includes(requestTypeParsed) ? - 'objectPut' : requestTypeParsed; - const parsedMainApiCall = actionsToConsiderAsObjectPut.includes(mainApiCall) ? - 'objectPut' : mainApiCall; + let requestTypeParsed = requestType.endsWith('Version') ? requestType.slice(0, 'Version'.length * -1) : requestType; + requestTypeParsed = actionsToConsiderAsObjectPut.includes(requestTypeParsed) ? 'objectPut' : requestTypeParsed; + const parsedMainApiCall = actionsToConsiderAsObjectPut.includes(mainApiCall) ? 'objectPut' : mainApiCall; if (bucket.getOwner() === canonicalID) { return true; } @@ -87,64 +83,66 @@ function checkBucketAcls(bucket, requestType, canonicalID, mainApiCall) { const bucketAcl = bucket.getAcl(); if (requestTypeParsed === 'bucketGet' || requestTypeParsed === 'bucketHead') { - if (bucketAcl.Canned === 'public-read' - || bucketAcl.Canned === 'public-read-write' - || (bucketAcl.Canned === 'authenticated-read' - && canonicalID !== publicId)) { + if ( + bucketAcl.Canned === 'public-read' || + bucketAcl.Canned === 'public-read-write' || + (bucketAcl.Canned === 'authenticated-read' && canonicalID !== publicId) + ) { return true; - } else if (bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1 - || bucketAcl.READ.indexOf(canonicalID) > -1) { + } else if (bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1 || bucketAcl.READ.indexOf(canonicalID) > -1) { return true; - } else if (bucketAcl.READ.indexOf(publicId) > -1 - || (bucketAcl.READ.indexOf(allAuthedUsersId) > -1 - && canonicalID !== publicId) - || (bucketAcl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1 - && canonicalID !== publicId) - || bucketAcl.FULL_CONTROL.indexOf(publicId) > -1) { + } else if ( + bucketAcl.READ.indexOf(publicId) > -1 || + (bucketAcl.READ.indexOf(allAuthedUsersId) > -1 && canonicalID !== publicId) || + (bucketAcl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1 && canonicalID !== publicId) || + bucketAcl.FULL_CONTROL.indexOf(publicId) > -1 + ) { return true; } } if (requestTypeParsed === 'bucketGetACL') { - if ((bucketAcl.Canned === 'log-delivery-write' - && canonicalID === logId) - || bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1 - || bucketAcl.READ_ACP.indexOf(canonicalID) > -1) { + if ( + (bucketAcl.Canned === 'log-delivery-write' && canonicalID === logId) || + bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1 || + bucketAcl.READ_ACP.indexOf(canonicalID) > -1 + ) { return true; - } else if (bucketAcl.READ_ACP.indexOf(publicId) > -1 - || (bucketAcl.READ_ACP.indexOf(allAuthedUsersId) > -1 - && canonicalID !== publicId) - || (bucketAcl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1 - && canonicalID !== publicId) - || bucketAcl.FULL_CONTROL.indexOf(publicId) > -1) { + } else if ( + bucketAcl.READ_ACP.indexOf(publicId) > -1 || + (bucketAcl.READ_ACP.indexOf(allAuthedUsersId) > -1 && canonicalID !== publicId) || + (bucketAcl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1 && canonicalID !== publicId) || + bucketAcl.FULL_CONTROL.indexOf(publicId) > -1 + ) { return true; } } if (requestTypeParsed === 'bucketPutACL') { - if (bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1 - || bucketAcl.WRITE_ACP.indexOf(canonicalID) > -1) { + if (bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1 || bucketAcl.WRITE_ACP.indexOf(canonicalID) > -1) { return true; - } else if (bucketAcl.WRITE_ACP.indexOf(publicId) > -1 - || (bucketAcl.WRITE_ACP.indexOf(allAuthedUsersId) > -1 - && canonicalID !== publicId) - || (bucketAcl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1 - && canonicalID !== publicId) - || bucketAcl.FULL_CONTROL.indexOf(publicId) > -1) { + } else if ( + bucketAcl.WRITE_ACP.indexOf(publicId) > -1 || + (bucketAcl.WRITE_ACP.indexOf(allAuthedUsersId) > -1 && canonicalID !== publicId) || + (bucketAcl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1 && canonicalID !== publicId) || + bucketAcl.FULL_CONTROL.indexOf(publicId) > -1 + ) { return true; } } if (requestTypeParsed === 'objectDelete' || requestTypeParsed === 'objectPut') { - if (bucketAcl.Canned === 'public-read-write' - || bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1 - || bucketAcl.WRITE.indexOf(canonicalID) > -1) { + if ( + bucketAcl.Canned === 'public-read-write' || + bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1 || + bucketAcl.WRITE.indexOf(canonicalID) > -1 + ) { return true; - } else if (bucketAcl.WRITE.indexOf(publicId) > -1 - || (bucketAcl.WRITE.indexOf(allAuthedUsersId) > -1 - && canonicalID !== publicId) - || (bucketAcl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1 - && canonicalID !== publicId) - || bucketAcl.FULL_CONTROL.indexOf(publicId) > -1) { + } else if ( + bucketAcl.WRITE.indexOf(publicId) > -1 || + (bucketAcl.WRITE.indexOf(allAuthedUsersId) > -1 && canonicalID !== publicId) || + (bucketAcl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1 && canonicalID !== publicId) || + bucketAcl.FULL_CONTROL.indexOf(publicId) > -1 + ) { return true; } } @@ -154,20 +152,28 @@ function checkBucketAcls(bucket, requestType, canonicalID, mainApiCall) { // objectPutACL, objectGetACL, objectHead or objectGet, the bucket // authorization check should just return true so can move on to check // rights at the object level. - return (requestTypeParsed === 'objectPutACL' || requestTypeParsed === 'objectGetACL' - || requestTypeParsed === 'objectGet' || requestTypeParsed === 'objectHead'); + return ( + requestTypeParsed === 'objectPutACL' || + requestTypeParsed === 'objectGetACL' || + requestTypeParsed === 'objectGet' || + requestTypeParsed === 'objectHead' + ); } -function checkObjectAcls(bucket, objectMD, requestType, canonicalID, requesterIsNotUser, - isUserUnauthenticated, mainApiCall) { +function checkObjectAcls( + bucket, + objectMD, + requestType, + canonicalID, + requesterIsNotUser, + isUserUnauthenticated, + mainApiCall, +) { const bucketOwner = bucket.getOwner(); - const requestTypeParsed = actionsToConsiderAsObjectPut.includes(requestType) ? - 'objectPut' : requestType; - const parsedMainApiCall = actionsToConsiderAsObjectPut.includes(mainApiCall) ? - 'objectPut' : mainApiCall; + const requestTypeParsed = actionsToConsiderAsObjectPut.includes(requestType) ? 'objectPut' : requestType; + const parsedMainApiCall = actionsToConsiderAsObjectPut.includes(mainApiCall) ? 'objectPut' : mainApiCall; // acls don't distinguish between users and accounts, so both should be allowed - if (bucketOwnerActions.includes(requestTypeParsed) - && (bucketOwner === canonicalID)) { + if (bucketOwnerActions.includes(requestTypeParsed) && bucketOwner === canonicalID) { return true; } if (objectMD['owner-id'] === canonicalID) { @@ -176,8 +182,10 @@ function checkObjectAcls(bucket, objectMD, requestType, canonicalID, requesterIs // Backward compatibility if (parsedMainApiCall === 'objectGet') { - if ((isUserUnauthenticated || (requesterIsNotUser && bucketOwner === objectMD['owner-id'])) - && requestTypeParsed === 'objectGetTagging') { + if ( + (isUserUnauthenticated || (requesterIsNotUser && bucketOwner === objectMD['owner-id'])) && + requestTypeParsed === 'objectGetTagging' + ) { return true; } } @@ -187,25 +195,26 @@ function checkObjectAcls(bucket, objectMD, requestType, canonicalID, requesterIs } if (requestTypeParsed === 'objectGet' || requestTypeParsed === 'objectHead') { - if (objectMD.acl.Canned === 'public-read' - || objectMD.acl.Canned === 'public-read-write' - || (objectMD.acl.Canned === 'authenticated-read' - && canonicalID !== publicId)) { + if ( + objectMD.acl.Canned === 'public-read' || + objectMD.acl.Canned === 'public-read-write' || + (objectMD.acl.Canned === 'authenticated-read' && canonicalID !== publicId) + ) { return true; - } else if (objectMD.acl.Canned === 'bucket-owner-read' - && bucketOwner === canonicalID) { + } else if (objectMD.acl.Canned === 'bucket-owner-read' && bucketOwner === canonicalID) { return true; - } else if ((objectMD.acl.Canned === 'bucket-owner-full-control' - && bucketOwner === canonicalID) - || objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1 - || objectMD.acl.READ.indexOf(canonicalID) > -1) { + } else if ( + (objectMD.acl.Canned === 'bucket-owner-full-control' && bucketOwner === canonicalID) || + objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1 || + objectMD.acl.READ.indexOf(canonicalID) > -1 + ) { return true; - } else if (objectMD.acl.READ.indexOf(publicId) > -1 - || (objectMD.acl.READ.indexOf(allAuthedUsersId) > -1 - && canonicalID !== publicId) - || (objectMD.acl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1 - && canonicalID !== publicId) - || objectMD.acl.FULL_CONTROL.indexOf(publicId) > -1) { + } else if ( + objectMD.acl.READ.indexOf(publicId) > -1 || + (objectMD.acl.READ.indexOf(allAuthedUsersId) > -1 && canonicalID !== publicId) || + (objectMD.acl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1 && canonicalID !== publicId) || + objectMD.acl.FULL_CONTROL.indexOf(publicId) > -1 + ) { return true; } } @@ -217,33 +226,35 @@ function checkObjectAcls(bucket, objectMD, requestType, canonicalID, requesterIs } if (requestTypeParsed === 'objectPutACL') { - if ((objectMD.acl.Canned === 'bucket-owner-full-control' - && bucketOwner === canonicalID) - || objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1 - || objectMD.acl.WRITE_ACP.indexOf(canonicalID) > -1) { + if ( + (objectMD.acl.Canned === 'bucket-owner-full-control' && bucketOwner === canonicalID) || + objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1 || + objectMD.acl.WRITE_ACP.indexOf(canonicalID) > -1 + ) { return true; - } else if (objectMD.acl.WRITE_ACP.indexOf(publicId) > -1 - || (objectMD.acl.WRITE_ACP.indexOf(allAuthedUsersId) > -1 - && canonicalID !== publicId) - || (objectMD.acl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1 - && canonicalID !== publicId) - || objectMD.acl.FULL_CONTROL.indexOf(publicId) > -1) { + } else if ( + objectMD.acl.WRITE_ACP.indexOf(publicId) > -1 || + (objectMD.acl.WRITE_ACP.indexOf(allAuthedUsersId) > -1 && canonicalID !== publicId) || + (objectMD.acl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1 && canonicalID !== publicId) || + objectMD.acl.FULL_CONTROL.indexOf(publicId) > -1 + ) { return true; } } if (requestTypeParsed === 'objectGetACL') { - if ((objectMD.acl.Canned === 'bucket-owner-full-control' - && bucketOwner === canonicalID) - || objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1 - || objectMD.acl.READ_ACP.indexOf(canonicalID) > -1) { + if ( + (objectMD.acl.Canned === 'bucket-owner-full-control' && bucketOwner === canonicalID) || + objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1 || + objectMD.acl.READ_ACP.indexOf(canonicalID) > -1 + ) { return true; - } else if (objectMD.acl.READ_ACP.indexOf(publicId) > -1 - || (objectMD.acl.READ_ACP.indexOf(allAuthedUsersId) > -1 - && canonicalID !== publicId) - || (objectMD.acl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1 - && canonicalID !== publicId) - || objectMD.acl.FULL_CONTROL.indexOf(publicId) > -1) { + } else if ( + objectMD.acl.READ_ACP.indexOf(publicId) > -1 || + (objectMD.acl.READ_ACP.indexOf(allAuthedUsersId) > -1 && canonicalID !== publicId) || + (objectMD.acl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1 && canonicalID !== publicId) || + objectMD.acl.FULL_CONTROL.indexOf(publicId) > -1 + ) { return true; } } @@ -251,9 +262,10 @@ function checkObjectAcls(bucket, objectMD, requestType, canonicalID, requesterIs // allow public reads on buckets that are whitelisted for anonymous reads // TODO: remove this after bucket policies are implemented const bucketAcl = bucket.getAcl(); - const allowPublicReads = publicReadBuckets.includes(bucket.getName()) - && bucketAcl.Canned === 'public-read' - && (requestTypeParsed === 'objectGet' || requestTypeParsed === 'objectHead'); + const allowPublicReads = + publicReadBuckets.includes(bucket.getName()) && + bucketAcl.Canned === 'public-read' && + (requestTypeParsed === 'objectGet' || requestTypeParsed === 'objectHead'); if (allowPublicReads) { return true; } @@ -289,7 +301,7 @@ function _getAccountId(arn) { } function _isAccountId(principal) { - return (principal.length === 12 && /^\d+$/.test(principal)); + return principal.length === 12 && /^\d+$/.test(principal); } /** @@ -329,15 +341,17 @@ function _checkCrossAccount(requesterARN, requesterCanonicalID, bucketOwnerCanon // Vault returns ARNs like 'arn:aws:iam::123456789012:/accountName/' for root accounts // with an empty resource type (missing 'user/' prefix) if (!_isRootUser(requesterARN)) { - return bucketOwnerCanonicalID === requesterCanonicalID ? - checkPrincipalResult.OK : checkPrincipalResult.CROSS_ACCOUNT_OK; + return bucketOwnerCanonicalID === requesterCanonicalID + ? checkPrincipalResult.OK + : checkPrincipalResult.CROSS_ACCOUNT_OK; } return checkPrincipalResult.OK; } function _checkPrincipalWildcard(requestARN, requesterCanonicalID, bucketOwnerCanonicalID) { - if (requestARN === undefined) { // User in unauthenticated (anonymous request) + if (requestARN === undefined) { + // User in unauthenticated (anonymous request) return checkPrincipalResult.OK; } @@ -349,7 +363,8 @@ function _checkPrincipalAWS(principal, requesterARN, requesterCanonicalID, bucke return _checkPrincipalWildcard(requesterARN, requesterCanonicalID, bucketOwnerCanonicalID); } - if (requesterARN === undefined) { // User in unauthenticated (anonymous request) + if (requesterARN === undefined) { + // User in unauthenticated (anonymous request) return checkPrincipalResult.KO; } @@ -373,7 +388,8 @@ function _checkPrincipalCanonicalUser(principal, requesterARN, requesterCanonica return _checkPrincipalWildcard(requesterARN, requesterCanonicalID, bucketOwnerCanonicalID); } - if (requesterARN === undefined) { // User in unauthenticated (anonymous request) + if (requesterARN === undefined) { + // User in unauthenticated (anonymous request) return checkPrincipalResult.KO; } @@ -411,13 +427,15 @@ function _checkPrincipals(canonicalID, arn, principal, bucketOwnerCanonicalID) { } if (principal.CanonicalUser) { - return _findBestPrincipalMatch(principal.CanonicalUser, - p => _checkPrincipalCanonicalUser(p, arn, canonicalID, bucketOwnerCanonicalID)); + return _findBestPrincipalMatch(principal.CanonicalUser, p => + _checkPrincipalCanonicalUser(p, arn, canonicalID, bucketOwnerCanonicalID), + ); } if (principal.AWS) { - return _findBestPrincipalMatch(principal.AWS, - p => _checkPrincipalAWS(p, arn, canonicalID, bucketOwnerCanonicalID)); + return _findBestPrincipalMatch(principal.AWS, p => + _checkPrincipalAWS(p, arn, canonicalID, bucketOwnerCanonicalID), + ); } return checkPrincipalResult.KO; @@ -456,11 +474,30 @@ function checkBucketPolicy(policy, requestType, canonicalID, arn, bucketOwner, l const ip = request ? requestUtils.getClientIp(request, config) : undefined; const isSecure = request ? requestUtils.getHttpProtocolSecurity(request, config) : undefined; - const requestContext = request ? new RequestContext(request.headers, request.query, - request.bucketName, request.objectKey, ip, - isSecure, request.resourceType, 's3', null, null, - null, null, null, null, null, null, null, null, null, - request.objectLockRetentionDays) : undefined; + const requestContext = request + ? new RequestContext( + request.headers, + request.query, + request.bucketName, + request.objectKey, + ip, + isSecure, + request.resourceType, + 's3', + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + request.objectLockRetentionDays, + ) + : undefined; while (copiedStatement.length > 0) { const s = copiedStatement[0]; @@ -470,34 +507,34 @@ function checkBucketPolicy(policy, requestType, canonicalID, arn, bucketOwner, l const conditionsMatch = _checkBucketPolicyConditions(requestContext, s.Condition, log); const ok = principalMatch === checkPrincipalResult.OK && actionMatch && resourceMatch && conditionsMatch; - const okCross = principalMatch === checkPrincipalResult.CROSS_ACCOUNT_OK - && actionMatch && resourceMatch && conditionsMatch; + const okCross = + principalMatch === checkPrincipalResult.CROSS_ACCOUNT_OK && actionMatch && resourceMatch && conditionsMatch; switch (permission) { - case checkBucketPolicyResult.DEFAULT_DENY: - if ((ok || okCross) && s.Effect === 'Deny') { - return checkBucketPolicyResult.EXPLICIT_DENY; - } else if (ok && s.Effect === 'Allow') { - permission = checkBucketPolicyResult.ALLOW; - } else if (okCross && s.Effect === 'Allow') { - permission = checkBucketPolicyResult.CROSS_ACCOUNT_ALLOW; - } - break; - case checkBucketPolicyResult.EXPLICIT_DENY: - return checkBucketPolicyResult.EXPLICIT_DENY; - case checkBucketPolicyResult.ALLOW: - if ((ok || okCross) && s.Effect === 'Deny') { - return checkBucketPolicyResult.EXPLICIT_DENY; - } - break; - case checkBucketPolicyResult.CROSS_ACCOUNT_ALLOW: - if ((ok || okCross) && s.Effect === 'Deny') { + case checkBucketPolicyResult.DEFAULT_DENY: + if ((ok || okCross) && s.Effect === 'Deny') { + return checkBucketPolicyResult.EXPLICIT_DENY; + } else if (ok && s.Effect === 'Allow') { + permission = checkBucketPolicyResult.ALLOW; + } else if (okCross && s.Effect === 'Allow') { + permission = checkBucketPolicyResult.CROSS_ACCOUNT_ALLOW; + } + break; + case checkBucketPolicyResult.EXPLICIT_DENY: return checkBucketPolicyResult.EXPLICIT_DENY; - } else if (ok && s.Effect === 'Allow') { - permission = checkBucketPolicyResult.ALLOW; - } - break; - default: // Needed for the linter, should be unreachable. - break; + case checkBucketPolicyResult.ALLOW: + if ((ok || okCross) && s.Effect === 'Deny') { + return checkBucketPolicyResult.EXPLICIT_DENY; + } + break; + case checkBucketPolicyResult.CROSS_ACCOUNT_ALLOW: + if ((ok || okCross) && s.Effect === 'Deny') { + return checkBucketPolicyResult.EXPLICIT_DENY; + } else if (ok && s.Effect === 'Allow') { + permission = checkBucketPolicyResult.ALLOW; + } + break; + default: // Needed for the linter, should be unreachable. + break; } copiedStatement = copiedStatement.splice(1); @@ -505,8 +542,18 @@ function checkBucketPolicy(policy, requestType, canonicalID, arn, bucketOwner, l return permission; } -function processBucketPolicy(requestType, bucket, canonicalID, arn, bucketOwner, log, - request, aclPermission, results, actionImplicitDenies) { +function processBucketPolicy( + requestType, + bucket, + canonicalID, + arn, + bucketOwner, + log, + request, + aclPermission, + results, + actionImplicitDenies, +) { const bucketPolicy = bucket.getBucketPolicy(); let processedResult = results[requestType]; let aclRequired = false; @@ -514,15 +561,25 @@ function processBucketPolicy(requestType, bucket, canonicalID, arn, bucketOwner, processedResult = actionImplicitDenies[requestType] === false && aclPermission; aclRequired = true; } else { - const bucketPolicyPermission = checkBucketPolicy(bucketPolicy, requestType, canonicalID, arn, - bucketOwner, log, request, actionImplicitDenies); + const bucketPolicyPermission = checkBucketPolicy( + bucketPolicy, + requestType, + canonicalID, + arn, + bucketOwner, + log, + request, + actionImplicitDenies, + ); if (bucketPolicyPermission === checkBucketPolicyResult.EXPLICIT_DENY) { processedResult = false; } else if (bucketPolicyPermission === checkBucketPolicyResult.ALLOW) { processedResult = true; - } else if (bucketPolicyPermission === checkBucketPolicyResult.CROSS_ACCOUNT_ALLOW - && actionImplicitDenies[requestType] === false) { + } else if ( + bucketPolicyPermission === checkBucketPolicyResult.CROSS_ACCOUNT_ALLOW && + actionImplicitDenies[requestType] === false + ) { // If the bucket policy is cross account, only return true if Vault also returned an explicit allow. processedResult = true; } else { @@ -533,8 +590,16 @@ function processBucketPolicy(requestType, bucket, canonicalID, arn, bucketOwner, return { allowed: processedResult, aclRequired }; } -function isBucketAuthorized(bucket, requestTypesInput, canonicalID, authInfo, log, request, - actionImplicitDeniesInput = {}, isWebsite = false) { +function isBucketAuthorized( + bucket, + requestTypesInput, + canonicalID, + authInfo, + log, + request, + actionImplicitDeniesInput = {}, + isWebsite = false, +) { const requestTypes = Array.isArray(requestTypesInput) ? requestTypesInput : [requestTypesInput]; const actionImplicitDenies = !actionImplicitDeniesInput ? {} : actionImplicitDeniesInput; const mainApiCall = requestTypes[0]; @@ -553,7 +618,7 @@ function isBucketAuthorized(bucket, requestTypesInput, canonicalID, authInfo, lo arn = authInfo.getArn(); } // if the bucket owner is an account, users should not have default access - if ((bucket.getOwner() === canonicalID) && requesterIsNotUser || isServiceAccount(canonicalID)) { + if ((bucket.getOwner() === canonicalID && requesterIsNotUser) || isServiceAccount(canonicalID)) { results[_requestType] = actionImplicitDenies[_requestType] === false; return results[_requestType]; } @@ -567,8 +632,18 @@ function isBucketAuthorized(bucket, requestTypesInput, canonicalID, authInfo, lo _requestType = 'objectGet'; actionImplicitDenies.objectGet = actionImplicitDenies.objectGet || false; } - const { allowed, aclRequired } = processBucketPolicy(_requestType, bucket, canonicalID, arn, - bucket.getOwner(), log, request, aclPermission, results, actionImplicitDenies); + const { allowed, aclRequired } = processBucketPolicy( + _requestType, + bucket, + canonicalID, + arn, + bucket.getOwner(), + log, + request, + aclPermission, + results, + actionImplicitDenies, + ); if (aclRequired && request?.serverAccessLog) { // eslint-disable-next-line no-param-reassign request.serverAccessLog.aclRequired = 'Yes'; @@ -577,8 +652,15 @@ function isBucketAuthorized(bucket, requestTypesInput, canonicalID, authInfo, lo }); } -function evaluateBucketPolicyWithIAM(bucket, requestTypesInput, canonicalID, authInfo, actionImplicitDeniesInput = {}, - log, request) { +function evaluateBucketPolicyWithIAM( + bucket, + requestTypesInput, + canonicalID, + authInfo, + actionImplicitDeniesInput = {}, + log, + request, +) { const requestTypes = Array.isArray(requestTypesInput) ? requestTypesInput : [requestTypesInput]; const actionImplicitDenies = !actionImplicitDeniesInput ? {} : actionImplicitDeniesInput; const results = {}; @@ -590,14 +672,33 @@ function evaluateBucketPolicyWithIAM(bucket, requestTypesInput, canonicalID, aut if (authInfo) { arn = authInfo.getArn(); } - const { allowed } = processBucketPolicy(_requestType, bucket, canonicalID, arn, bucket.getOwner(), log, - request, true, results, actionImplicitDenies); + const { allowed } = processBucketPolicy( + _requestType, + bucket, + canonicalID, + arn, + bucket.getOwner(), + log, + request, + true, + results, + actionImplicitDenies, + ); return allowed; }); } -function isObjAuthorized(bucket, objectMD, requestTypesInput, canonicalID, authInfo, log, request, - actionImplicitDeniesInput = {}, isWebsite = false) { +function isObjAuthorized( + bucket, + objectMD, + requestTypesInput, + canonicalID, + authInfo, + log, + request, + actionImplicitDeniesInput = {}, + isWebsite = false, +) { const requestTypes = Array.isArray(requestTypesInput) ? requestTypesInput : [requestTypesInput]; const actionImplicitDenies = !actionImplicitDeniesInput ? {} : actionImplicitDeniesInput; const results = {}; @@ -606,8 +707,7 @@ function isObjAuthorized(bucket, objectMD, requestTypesInput, canonicalID, authI // By default, all missing actions are defined as allowed from IAM, to be // backward compatible actionImplicitDenies[_requestType] = actionImplicitDenies[_requestType] || false; - const parsedMethodName = _requestType.endsWith('Version') - ? _requestType.slice(0, -7) : _requestType; + const parsedMethodName = _requestType.endsWith('Version') ? _requestType.slice(0, -7) : _requestType; const bucketOwner = bucket.getOwner(); if (!objectMD) { // check bucket has read access @@ -616,12 +716,22 @@ function isObjAuthorized(bucket, objectMD, requestTypesInput, canonicalID, authI if (actionsToConsiderAsObjectPut.includes(_requestType)) { permission = 'objectPut'; } - results[_requestType] = isBucketAuthorized(bucket, permission, canonicalID, authInfo, log, request, - actionImplicitDenies, isWebsite); + results[_requestType] = isBucketAuthorized( + bucket, + permission, + canonicalID, + authInfo, + log, + request, + actionImplicitDenies, + isWebsite, + ); // User is already authorized on the bucket for FULL_CONTROL or WRITE or // bucket has canned ACL public-read-write - if ((parsedMethodName === 'objectPut' || parsedMethodName === 'objectDelete') - && results[_requestType] === false) { + if ( + (parsedMethodName === 'objectPut' || parsedMethodName === 'objectDelete') && + results[_requestType] === false + ) { results[_requestType] = actionImplicitDenies[_requestType] === false; } return results[_requestType]; @@ -634,7 +744,7 @@ function isObjAuthorized(bucket, objectMD, requestTypesInput, canonicalID, authI arn = authInfo.getArn(); isUserUnauthenticated = arn === undefined; } - if (objectMD['owner-id'] === canonicalID && requesterIsNotUser || isServiceAccount(canonicalID)) { + if ((objectMD['owner-id'] === canonicalID && requesterIsNotUser) || isServiceAccount(canonicalID)) { results[_requestType] = actionImplicitDenies[_requestType] === false; return results[_requestType]; } @@ -642,16 +752,31 @@ function isObjAuthorized(bucket, objectMD, requestTypesInput, canonicalID, authI // - requesttype is included in bucketOwnerActions and // - account is the bucket owner // - requester is account, not user - if (bucketOwnerActions.includes(parsedMethodName) - && (bucketOwner === canonicalID) - && requesterIsNotUser) { + if (bucketOwnerActions.includes(parsedMethodName) && bucketOwner === canonicalID && requesterIsNotUser) { results[_requestType] = actionImplicitDenies[_requestType] === false; return results[_requestType]; } - const aclPermission = checkObjectAcls(bucket, objectMD, parsedMethodName, - canonicalID, requesterIsNotUser, isUserUnauthenticated, mainApiCall); - const { allowed, aclRequired } = processBucketPolicy(_requestType, bucket, canonicalID, arn, bucketOwner, - log, request, aclPermission, results, actionImplicitDenies); + const aclPermission = checkObjectAcls( + bucket, + objectMD, + parsedMethodName, + canonicalID, + requesterIsNotUser, + isUserUnauthenticated, + mainApiCall, + ); + const { allowed, aclRequired } = processBucketPolicy( + _requestType, + bucket, + canonicalID, + arn, + bucketOwner, + log, + request, + aclPermission, + results, + actionImplicitDenies, + ); if (aclRequired && request?.serverAccessLog) { // eslint-disable-next-line no-param-reassign request.serverAccessLog.aclRequired = 'Yes'; @@ -752,8 +877,8 @@ function validatePolicyConditions(policy) { for (const conditionOperator of conditionOperators) { const conditionKey = Object.keys(s.Condition[conditionOperator])[0]; const conditionValue = s.Condition[conditionOperator][conditionKey]; - const validCondition = validConditions.find(validCondition => - validCondition.conditionKey === conditionKey + const validCondition = validConditions.find( + validCondition => validCondition.conditionKey === conditionKey, ); // AWS returns does not return an error if the condition starts with 'aws:' // so we reproduce this behaviour @@ -772,7 +897,6 @@ function validatePolicyConditions(policy) { return null; } - /** isLifecycleSession - check if it is the Lifecycle assumed role session arn. * @param {string} arn - Amazon resource name - example: * arn:aws:sts::257038443293:assumed-role/rolename/backbeat-lifecycle @@ -791,9 +915,9 @@ function isLifecycleSession(arn) { const resourceType = resourceNames[0]; const sessionName = resourceNames[resourceNames.length - 1]; - return (service === 'sts' - && resourceType === assumedRoleArnResourceType - && sessionName === backbeatLifecycleSessionName); + return ( + service === 'sts' && resourceType === assumedRoleArnResourceType && sessionName === backbeatLifecycleSessionName + ); } module.exports = { diff --git a/lib/api/apiUtils/authorization/prepareRequestContexts.js b/lib/api/apiUtils/authorization/prepareRequestContexts.js index 4fcf504aeb..1b7d58b610 100644 --- a/lib/api/apiUtils/authorization/prepareRequestContexts.js +++ b/lib/api/apiUtils/authorization/prepareRequestContexts.js @@ -18,9 +18,13 @@ const apiMethodWithVersion = { }; function isHeaderAcl(headers) { - return headers['x-amz-grant-read'] || headers['x-amz-grant-read-acp'] || - headers['x-amz-grant-write-acp'] || headers['x-amz-grant-full-control'] || - headers['x-amz-acl']; + return ( + headers['x-amz-grant-read'] || + headers['x-amz-grant-read-acp'] || + headers['x-amz-grant-write-acp'] || + headers['x-amz-grant-full-control'] || + headers['x-amz-acl'] + ); } /** @@ -32,8 +36,7 @@ function isHeaderAcl(headers) { * @param {string} sourceVersionId - value of sourceVersionId if copy request * @return {RequestContext []} array of requestContexts */ -function prepareRequestContexts(apiMethod, request, sourceBucket, - sourceObject, sourceVersionId) { +function prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId) { // if multiObjectDelete request, we want to authenticate // before parsing the post body and creating multiple requestContexts // so send null as requestContexts to Vault to avoid authorization @@ -48,17 +51,23 @@ function prepareRequestContexts(apiMethod, request, sourceBucket, const isSecure = requestUtils.getHttpProtocolSecurity(request, config); function generateRequestContext(apiMethod) { - return new RequestContext(request.headers, - request.query, request.bucketName, request.objectKey, - ip, isSecure, apiMethod, 's3'); + return new RequestContext( + request.headers, + request.query, + request.bucketName, + request.objectKey, + ip, + isSecure, + apiMethod, + 's3', + ); } if (apiMethod === 'bucketPut') { return null; } - if (apiMethodWithVersion[apiMethod] && request.query && - request.query.versionId) { + if (apiMethodWithVersion[apiMethod] && request.query && request.query.versionId) { apiMethodAfterVersionCheck = `${apiMethod}Version`; } else { apiMethodAfterVersionCheck = apiMethod; @@ -75,181 +84,153 @@ function prepareRequestContexts(apiMethod, request, sourceBucket, // In the API, we then ignore these authorization results, and we can use // any information returned, e.g., the quota. const requestContextMultiObjectDelete = generateRequestContext('objectDelete'); - requestContexts.push(requestContextMultiObjectDelete); - } else if (apiMethodAfterVersionCheck === 'objectCopy' - || apiMethodAfterVersionCheck === 'objectPutCopyPart') { - const objectGetAction = sourceVersionId ? 'objectGetVersion' : - 'objectGet'; - const reqQuery = Object.assign({}, request.query, - { versionId: sourceVersionId }); - const getRequestContext = new RequestContext(request.headers, - reqQuery, sourceBucket, sourceObject, - ip, isSecure, - objectGetAction, 's3'); + requestContexts.push(requestContextMultiObjectDelete); + } else if (apiMethodAfterVersionCheck === 'objectCopy' || apiMethodAfterVersionCheck === 'objectPutCopyPart') { + const objectGetAction = sourceVersionId ? 'objectGetVersion' : 'objectGet'; + const reqQuery = Object.assign({}, request.query, { versionId: sourceVersionId }); + const getRequestContext = new RequestContext( + request.headers, + reqQuery, + sourceBucket, + sourceObject, + ip, + isSecure, + objectGetAction, + 's3', + ); const putRequestContext = generateRequestContext('objectPut'); requestContexts.push(getRequestContext, putRequestContext); if (apiMethodAfterVersionCheck === 'objectCopy') { // if tagging directive is COPY, "s3:PutObjectTagging" don't need // to be included in the list of permitted actions in IAM policy - if (request.headers['x-amz-tagging'] && - request.headers['x-amz-tagging-directive'] === 'REPLACE') { - const putTaggingRequestContext = - generateRequestContext('objectPutTagging'); + if (request.headers['x-amz-tagging'] && request.headers['x-amz-tagging-directive'] === 'REPLACE') { + const putTaggingRequestContext = generateRequestContext('objectPutTagging'); requestContexts.push(putTaggingRequestContext); } if (isHeaderAcl(request.headers)) { - const putAclRequestContext = - generateRequestContext('objectPutACL'); + const putAclRequestContext = generateRequestContext('objectPutACL'); requestContexts.push(putAclRequestContext); } } - } else if (apiMethodAfterVersionCheck === 'objectGet' - || apiMethodAfterVersionCheck === 'objectGetVersion') { - const objectGetTaggingAction = (request.query && - request.query.versionId) ? 'objectGetTaggingVersion' : - 'objectGetTagging'; + } else if (apiMethodAfterVersionCheck === 'objectGet' || apiMethodAfterVersionCheck === 'objectGetVersion') { + const objectGetTaggingAction = + request.query && request.query.versionId ? 'objectGetTaggingVersion' : 'objectGetTagging'; if (request.headers['x-amz-version-id']) { const objectGetVersionAction = 'objectGetVersion'; - const getVersionResourceVersion = - generateRequestContext(objectGetVersionAction); + const getVersionResourceVersion = generateRequestContext(objectGetVersionAction); requestContexts.push(getVersionResourceVersion); } - const getRequestContext = - generateRequestContext(apiMethodAfterVersionCheck); - const getTaggingRequestContext = - generateRequestContext(objectGetTaggingAction); + const getRequestContext = generateRequestContext(apiMethodAfterVersionCheck); + const getTaggingRequestContext = generateRequestContext(objectGetTaggingAction); requestContexts.push(getRequestContext, getTaggingRequestContext); } else if (apiMethodAfterVersionCheck === 'objectGetTagging') { const objectGetTaggingAction = 'objectGetTagging'; - const getTaggingResourceVersion = - generateRequestContext(objectGetTaggingAction); + const getTaggingResourceVersion = generateRequestContext(objectGetTaggingAction); requestContexts.push(getTaggingResourceVersion); if (request.headers['x-amz-version-id']) { const objectGetTaggingVersionAction = 'objectGetTaggingVersion'; - const getTaggingVersionResourceVersion = - generateRequestContext(objectGetTaggingVersionAction); + const getTaggingVersionResourceVersion = generateRequestContext(objectGetTaggingVersionAction); requestContexts.push(getTaggingVersionResourceVersion); } } else if (apiMethodAfterVersionCheck === 'objectHead') { const objectHeadAction = 'objectHead'; - const headObjectAction = - generateRequestContext(objectHeadAction); + const headObjectAction = generateRequestContext(objectHeadAction); requestContexts.push(headObjectAction); if (request.headers['x-amz-version-id']) { const objectHeadVersionAction = 'objectGetVersion'; - const headObjectVersion = - generateRequestContext(objectHeadVersionAction); + const headObjectVersion = generateRequestContext(objectHeadVersionAction); requestContexts.push(headObjectVersion); } if (request.headers['x-amz-scal-archive-info']) { - const coldStatus = - generateRequestContext('objectGetArchiveInfo'); + const coldStatus = generateRequestContext('objectGetArchiveInfo'); requestContexts.push(coldStatus); } } else if (apiMethodAfterVersionCheck === 'objectPutTagging') { - const putObjectTaggingRequestContext = - generateRequestContext('objectPutTagging'); + const putObjectTaggingRequestContext = generateRequestContext('objectPutTagging'); requestContexts.push(putObjectTaggingRequestContext); if (request.headers['x-amz-version-id']) { - const putObjectVersionRequestContext = - generateRequestContext('objectPutTaggingVersion'); + const putObjectVersionRequestContext = generateRequestContext('objectPutTaggingVersion'); requestContexts.push(putObjectVersionRequestContext); } } else if (apiMethodAfterVersionCheck === 'objectPut') { // if put object with version - if (request.headers['x-scal-s3-version-id'] || - request.headers['x-scal-s3-version-id'] === '') { - const putVersionRequestContext = - generateRequestContext('objectPutVersion'); + if (request.headers['x-scal-s3-version-id'] || request.headers['x-scal-s3-version-id'] === '') { + const putVersionRequestContext = generateRequestContext('objectPutVersion'); requestContexts.push(putVersionRequestContext); } else { - const putRequestContext = - generateRequestContext(apiMethodAfterVersionCheck); + const putRequestContext = generateRequestContext(apiMethodAfterVersionCheck); requestContexts.push(putRequestContext); // if put object (versioning) with tag set if (request.headers['x-amz-tagging']) { - const putTaggingRequestContext = - generateRequestContext('objectPutTagging'); + const putTaggingRequestContext = generateRequestContext('objectPutTagging'); requestContexts.push(putTaggingRequestContext); } if (['ON', 'OFF'].includes(request.headers['x-amz-object-lock-legal-hold-status'])) { - const putLegalHoldStatusAction = - generateRequestContext('objectPutLegalHold'); + const putLegalHoldStatusAction = generateRequestContext('objectPutLegalHold'); requestContexts.push(putLegalHoldStatusAction); } // if put object (versioning) with ACL if (isHeaderAcl(request.headers)) { - const putAclRequestContext = - generateRequestContext('objectPutACL'); + const putAclRequestContext = generateRequestContext('objectPutACL'); requestContexts.push(putAclRequestContext); } if (request.headers['x-amz-object-lock-mode']) { - const putObjectLockRequestContext = - generateRequestContext('objectPutRetention'); + const putObjectLockRequestContext = generateRequestContext('objectPutRetention'); requestContexts.push(putObjectLockRequestContext); if (hasGovernanceBypassHeader(request.headers)) { - const checkUserGovernanceBypassRequestContext = - generateRequestContext('bypassGovernanceRetention'); + const checkUserGovernanceBypassRequestContext = generateRequestContext('bypassGovernanceRetention'); requestContexts.push(checkUserGovernanceBypassRequestContext); } } if (request.headers['x-amz-version-id']) { - const putObjectVersionRequestContext = - generateRequestContext('objectPutTaggingVersion'); + const putObjectVersionRequestContext = generateRequestContext('objectPutTaggingVersion'); requestContexts.push(putObjectVersionRequestContext); } } - } else if (apiMethodAfterVersionCheck === 'objectPutRetention' || - apiMethodAfterVersionCheck === 'objectPutRetentionVersion') { - const putRetentionRequestContext = - generateRequestContext(apiMethodAfterVersionCheck); + } else if ( + apiMethodAfterVersionCheck === 'objectPutRetention' || + apiMethodAfterVersionCheck === 'objectPutRetentionVersion' + ) { + const putRetentionRequestContext = generateRequestContext(apiMethodAfterVersionCheck); requestContexts.push(putRetentionRequestContext); if (hasGovernanceBypassHeader(request.headers)) { - const checkUserGovernanceBypassRequestContext = - generateRequestContext('bypassGovernanceRetention'); + const checkUserGovernanceBypassRequestContext = generateRequestContext('bypassGovernanceRetention'); requestContexts.push(checkUserGovernanceBypassRequestContext); } - } else if (apiMethodAfterVersionCheck === 'initiateMultipartUpload' || - apiMethodAfterVersionCheck === 'objectPutPart' || - apiMethodAfterVersionCheck === 'completeMultipartUpload' - ) { - if (request.headers['x-scal-s3-version-id'] || - request.headers['x-scal-s3-version-id'] === '') { - const putVersionRequestContext = - generateRequestContext('objectPutVersion'); + } else if ( + apiMethodAfterVersionCheck === 'initiateMultipartUpload' || + apiMethodAfterVersionCheck === 'objectPutPart' || + apiMethodAfterVersionCheck === 'completeMultipartUpload' + ) { + if (request.headers['x-scal-s3-version-id'] || request.headers['x-scal-s3-version-id'] === '') { + const putVersionRequestContext = generateRequestContext('objectPutVersion'); requestContexts.push(putVersionRequestContext); } else { - const putRequestContext = - generateRequestContext(apiMethodAfterVersionCheck); + const putRequestContext = generateRequestContext(apiMethodAfterVersionCheck); requestContexts.push(putRequestContext); } // if put object (versioning) with ACL if (isHeaderAcl(request.headers)) { - const putAclRequestContext = - generateRequestContext('objectPutACL'); + const putAclRequestContext = generateRequestContext('objectPutACL'); requestContexts.push(putAclRequestContext); } if (request.headers['x-amz-object-lock-mode']) { - const putObjectLockRequestContext = - generateRequestContext('objectPutRetention'); + const putObjectLockRequestContext = generateRequestContext('objectPutRetention'); requestContexts.push(putObjectLockRequestContext); } if (request.headers['x-amz-version-id']) { - const putObjectVersionRequestContext = - generateRequestContext('objectPutTaggingVersion'); + const putObjectVersionRequestContext = generateRequestContext('objectPutTaggingVersion'); requestContexts.push(putObjectVersionRequestContext); } - // AWS only returns an object lock error if a version id - // is specified, else continue to create a delete marker + // AWS only returns an object lock error if a version id + // is specified, else continue to create a delete marker } else if (sourceVersionId && apiMethodAfterVersionCheck === 'objectDeleteVersion') { - const deleteRequestContext = - generateRequestContext(apiMethodAfterVersionCheck); + const deleteRequestContext = generateRequestContext(apiMethodAfterVersionCheck); requestContexts.push(deleteRequestContext); if (hasGovernanceBypassHeader(request.headers)) { - const checkUserGovernanceBypassRequestContext = - generateRequestContext('bypassGovernanceRetention'); + const checkUserGovernanceBypassRequestContext = generateRequestContext('bypassGovernanceRetention'); requestContexts.push(checkUserGovernanceBypassRequestContext); } } else if (apiMethodAfterVersionCheck === 'bucketGet') { @@ -268,10 +249,7 @@ function prepareRequestContexts(apiMethod, request, sourceBucket, generateRequestContext('objectGetVersionAttributes'), ); } else { - requestContexts.push( - generateRequestContext('objectGet'), - generateRequestContext('objectGetAttributes'), - ); + requestContexts.push(generateRequestContext('objectGet'), generateRequestContext('objectGetAttributes')); } const attributes = request.headers['x-amz-object-attributes']?.split(',') ?? []; @@ -279,8 +257,7 @@ function prepareRequestContexts(apiMethod, request, sourceBucket, requestContexts.push(generateRequestContext('objectGetAttributesCustom')); } } else { - const requestContext = - generateRequestContext(apiMethodAfterVersionCheck); + const requestContext = generateRequestContext(apiMethodAfterVersionCheck); requestContexts.push(requestContext); } diff --git a/lib/api/apiUtils/authorization/serviceUser.js b/lib/api/apiUtils/authorization/serviceUser.js index c3a125954d..743475b3cc 100644 --- a/lib/api/apiUtils/authorization/serviceUser.js +++ b/lib/api/apiUtils/authorization/serviceUser.js @@ -11,5 +11,5 @@ function isRateLimitServiceUser(authInfo) { } module.exports = { - isRateLimitServiceUser + isRateLimitServiceUser, }; diff --git a/lib/api/apiUtils/authorization/tagConditionKeys.js b/lib/api/apiUtils/authorization/tagConditionKeys.js index 47c09ce0b3..3bce314a11 100644 --- a/lib/api/apiUtils/authorization/tagConditionKeys.js +++ b/lib/api/apiUtils/authorization/tagConditionKeys.js @@ -13,36 +13,36 @@ function makeTagQuery(tags) { } function updateRequestContextsWithTags(request, requestContexts, apiMethod, log, cb) { - async.waterfall([ - next => { - if (request.headers['x-amz-tagging']) { - return next(null, request.headers['x-amz-tagging']); - } - if (request.post && apiMethod === 'objectPutTagging') { - return parseTagXml(request.post, log, (err, tags) => { - if (err) { - log.trace('error parsing request tags'); - return next(err); - } - return next(null, makeTagQuery(tags)); - }); - } - return next(null, null); - }, - (requestTagsQuery, next) => { - const objectKey = request.objectKey; - const bucketName = request.bucketName; - const decodedVidResult = decodeVersionId(request.query); - if (decodedVidResult instanceof Error) { - log.trace('invalid versionId query', { - versionId: request.query.versionId, - error: decodedVidResult, - }); - return next(decodedVidResult); - } - const reqVersionId = decodedVidResult; - return metadata.getObjectMD( - bucketName, objectKey, { versionId: reqVersionId }, log, (err, objMD) => { + async.waterfall( + [ + next => { + if (request.headers['x-amz-tagging']) { + return next(null, request.headers['x-amz-tagging']); + } + if (request.post && apiMethod === 'objectPutTagging') { + return parseTagXml(request.post, log, (err, tags) => { + if (err) { + log.trace('error parsing request tags'); + return next(err); + } + return next(null, makeTagQuery(tags)); + }); + } + return next(null, null); + }, + (requestTagsQuery, next) => { + const objectKey = request.objectKey; + const bucketName = request.bucketName; + const decodedVidResult = decodeVersionId(request.query); + if (decodedVidResult instanceof Error) { + log.trace('invalid versionId query', { + versionId: request.query.versionId, + error: decodedVidResult, + }); + return next(decodedVidResult); + } + const reqVersionId = decodedVidResult; + return metadata.getObjectMD(bucketName, objectKey, { versionId: reqVersionId }, log, (err, objMD) => { if (err) { // TODO: move to `.is` once BKTCLT-9 is done and bumped in Cloudserver if (err.NoSuchKey) { @@ -54,24 +54,26 @@ function updateRequestContextsWithTags(request, requestContexts, apiMethod, log, const existingTagsQuery = objMD.tags && makeTagQuery(objMD.tags); return next(null, requestTagsQuery, existingTagsQuery); }); - }, - ], (err, requestTagsQuery, existingTagsQuery) => { - if (err) { - log.trace('error processing tag condition key evaluation'); - return cb(err); - } - // FIXME introduced by CLDSRV-256, this syntax should be allowed by the linter - for (const rc of requestContexts) { - rc.setNeedTagEval(true); - if (requestTagsQuery) { - rc.setRequestObjTags(requestTagsQuery); + }, + ], + (err, requestTagsQuery, existingTagsQuery) => { + if (err) { + log.trace('error processing tag condition key evaluation'); + return cb(err); } - if (existingTagsQuery) { - rc.setExistingObjTag(existingTagsQuery); + // FIXME introduced by CLDSRV-256, this syntax should be allowed by the linter + for (const rc of requestContexts) { + rc.setNeedTagEval(true); + if (requestTagsQuery) { + rc.setRequestObjTags(requestTagsQuery); + } + if (existingTagsQuery) { + rc.setExistingObjTag(existingTagsQuery); + } } - } - return cb(); - }); + return cb(); + }, + ); } function tagConditionKeyAuth(authorizationResults, request, requestContexts, apiMethod, log, cb) { @@ -86,8 +88,13 @@ function tagConditionKeyAuth(authorizationResults, request, requestContexts, api if (err) { return cb(err); } - return auth.server.doAuth(request, log, - (err, userInfo, authResults) => cb(err, authResults), 's3', requestContexts); + return auth.server.doAuth( + request, + log, + (err, userInfo, authResults) => cb(err, authResults), + 's3', + requestContexts, + ); }); } diff --git a/lib/api/apiUtils/bucket/bucketCors.js b/lib/api/apiUtils/bucket/bucketCors.js index e81c3b1199..b76cce7f9b 100644 --- a/lib/api/apiUtils/bucket/bucketCors.js +++ b/lib/api/apiUtils/bucket/bucketCors.js @@ -26,10 +26,8 @@ const escapeForXml = s3middleware.escapeForXml; */ const customizedErrs = { - numberRules: 'The number of CORS rules should not exceed allowed limit ' + - 'of 100 rules.', - originAndMethodExist: 'Each CORSRule must identify at least one origin ' + - 'and one method.', + numberRules: 'The number of CORS rules should not exceed allowed limit ' + 'of 100 rules.', + originAndMethodExist: 'Each CORSRule must identify at least one origin ' + 'and one method.', }; // Helper validation methods @@ -42,21 +40,20 @@ const _validator = { validateNumberWildcards(string) { const firstIndex = string.indexOf('*'); if (firstIndex !== -1) { - return (string.indexOf('*', firstIndex + 1) === -1); + return string.indexOf('*', firstIndex + 1) === -1; } return true; }, /** _validator.validateID - check value of optional ID - * @param {string[]} id - array containing id string - * @return {(Error|true|undefined)} - Arsenal error on failure, true on - * success, undefined if ID does not exist - */ + * @param {string[]} id - array containing id string + * @return {(Error|true|undefined)} - Arsenal error on failure, true on + * success, undefined if ID does not exist + */ validateID(id) { if (!id) { return undefined; // to indicate ID does not exist } - if (!Array.isArray(id) || id.length !== 1 - || typeof id[0] !== 'string') { + if (!Array.isArray(id) || id.length !== 1 || typeof id[0] !== 'string') { return errors.MalformedXML; } if (id[0] === '') { @@ -65,10 +62,10 @@ const _validator = { return true; }, /** _validator.validateMaxAgeSeconds - check value of optional MaxAgeSeconds - * @param {string[]} seconds - array containing number string - * @return {(Error|parsedValue|undefined)} - Arsenal error on failure, parsed - * value if valid, undefined if MaxAgeSeconds does not exist - */ + * @param {string[]} seconds - array containing number string + * @return {(Error|parsedValue|undefined)} - Arsenal error on failure, parsed + * value if valid, undefined if MaxAgeSeconds does not exist + */ validateMaxAgeSeconds(seconds) { if (!seconds) { return undefined; @@ -87,36 +84,37 @@ const _validator = { return parsedValue; }, /** _validator.validateNumberRules - return if number of rules exceeds 100 - * @param {number} length - array containing number string - * @return {(Error|true)} - Arsenal error on failure, true on success - */ + * @param {number} length - array containing number string + * @return {(Error|true)} - Arsenal error on failure, true on success + */ validateNumberRules(length) { if (length > 100) { - return errorInstances.InvalidRequest - .customizeDescription(customizedErrs.numberRules); + return errorInstances.InvalidRequest.customizeDescription(customizedErrs.numberRules); } return true; }, /** _validator.validateOriginAndMethodExist - * @param {string[]} allowedMethods - array of AllowedMethod's - * @param {string[]} allowedOrigins - array of AllowedOrigin's - * @return {(Error|true)} - Arsenal error on failure, true on success - */ + * @param {string[]} allowedMethods - array of AllowedMethod's + * @param {string[]} allowedOrigins - array of AllowedOrigin's + * @return {(Error|true)} - Arsenal error on failure, true on success + */ validateOriginAndMethodExist(allowedMethods, allowedOrigins) { - if (allowedOrigins && allowedMethods && - Array.isArray(allowedOrigins) && - Array.isArray(allowedMethods) && - allowedOrigins.length > 0 && - allowedMethods.length > 0) { + if ( + allowedOrigins && + allowedMethods && + Array.isArray(allowedOrigins) && + Array.isArray(allowedMethods) && + allowedOrigins.length > 0 && + allowedMethods.length > 0 + ) { return true; } - return errorInstances.MalformedXML - .customizeDescription(customizedErrs.originAndMethodExist); + return errorInstances.MalformedXML.customizeDescription(customizedErrs.originAndMethodExist); }, /** _validator.validateMethods - check values of AllowedMethod's - * @param {string[]} methods - array of AllowedMethod's - * @return {(Error|true)} - Arsenal error on failure, true on success - */ + * @param {string[]} methods - array of AllowedMethod's + * @return {(Error|true)} - Arsenal error on failure, true on success + */ validateMethods(methods) { let invalidMethod; function isValidMethod(method) { @@ -128,17 +126,17 @@ const _validator = { return false; } if (!methods.every(isValidMethod)) { - const errMsg = 'Found unsupported HTTP method in CORS config. ' + - `Unsupported method is "${invalidMethod}"`; + const errMsg = + 'Found unsupported HTTP method in CORS config. ' + `Unsupported method is "${invalidMethod}"`; return errorInstances.InvalidRequest.customizeDescription(errMsg); } return true; }, /** _validator.validateAllowedOriginsOrHeaders - check values - * @param {string[]} elementArr - array of elements to check - * @param {string} typeElement - type of element being checked - * @return {(Error|true)} - Arsenal error on failure, true on success - */ + * @param {string[]} elementArr - array of elements to check + * @param {string} typeElement - type of element being checked + * @return {(Error|true)} - Arsenal error on failure, true on success + */ validateAllowedOriginsOrHeaders(elementArr, typeElement) { for (let i = 0; i < elementArr.length; i++) { const element = elementArr[i]; @@ -146,18 +144,17 @@ const _validator = { return errors.MalformedXML; } if (!this.validateNumberWildcards(element)) { - const errMsg = `${typeElement} "${element}" can not have ` + - 'more than one wildcard.'; + const errMsg = `${typeElement} "${element}" can not have ` + 'more than one wildcard.'; return errorInstances.InvalidRequest.customizeDescription(errMsg); } } return true; }, /** _validator.validateAllowedHeaders - check values of AllowedHeader's - * @param {string[]} headers - array of AllowedHeader's - * @return {(Error|true|undefined)} - Arsenal error on failure, true if - * valid, undefined if optional AllowedHeader's do not exist - */ + * @param {string[]} headers - array of AllowedHeader's + * @return {(Error|true|undefined)} - Arsenal error on failure, true if + * valid, undefined if optional AllowedHeader's do not exist + */ validateAllowedHeaders(headers) { if (!headers) { return undefined; // to indicate AllowedHeaders do not exist @@ -165,18 +162,17 @@ const _validator = { if (!Array.isArray(headers) || headers.length === 0) { return errors.MalformedXML; } - const result = - this.validateAllowedOriginsOrHeaders(headers, 'AllowedHeader'); + const result = this.validateAllowedOriginsOrHeaders(headers, 'AllowedHeader'); if (result instanceof Error) { return result; } return true; }, /** _validator.validateExposeHeaders - check values of ExposeHeader's - * @param {string[]} headers - array of ExposeHeader's - * @return {(Error|true|undefined)} - Arsenal error on failure, true if - * valid, undefined if optional ExposeHeader's do not exist - */ + * @param {string[]} headers - array of ExposeHeader's + * @return {(Error|true|undefined)} - Arsenal error on failure, true if + * valid, undefined if optional ExposeHeader's do not exist + */ validateExposeHeaders(headers) { if (!headers) { return undefined; // indicate ExposeHeaders do not exist @@ -190,13 +186,13 @@ const _validator = { return errors.MalformedXML; } if (header.indexOf('*') !== -1) { - const errMsg = `ExposeHeader ${header} contains a wildcard. ` + - 'Wildcards are currently not supported for ExposeHeader.'; + const errMsg = + `ExposeHeader ${header} contains a wildcard. ` + + 'Wildcards are currently not supported for ExposeHeader.'; return errorInstances.InvalidRequest.customizeDescription(errMsg); } if (!/^[A-Za-z0-9-]*$/.test(header)) { - const errMsg = `ExposeHeader ${header} contains invalid ` + - 'character.'; + const errMsg = `ExposeHeader ${header} contains invalid ` + 'character.'; return errorInstances.InvalidRequest.customizeDescription(errMsg); } } @@ -205,31 +201,29 @@ const _validator = { }; /** _validateCorsXml - Validate XML, returning an error if any part is not valid -* @param {object[]} rules - CORSRule collection parsed from xml to be validated -* @param {string[]} [rules[].ID] - optional id to identify rule -* @param {string[]} rules[].AllowedMethod - methods allowed for CORS -* @param {string[]} rules[].AllowedOrigin - origins allowed for CORS -* @param {string[]} [rules[].AllowedHeader] - headers allowed in an OPTIONS -* request via the Access-Control-Request-Headers header -* @param {string[]} [rules[].MaxAgeSeconds] - seconds browsers should cache -* OPTIONS response -* @param {string[]} [rules[].ExposeHeader] - headers exposed to applications -* @return {(Error|object)} - return cors object on success; error on failure -*/ + * @param {object[]} rules - CORSRule collection parsed from xml to be validated + * @param {string[]} [rules[].ID] - optional id to identify rule + * @param {string[]} rules[].AllowedMethod - methods allowed for CORS + * @param {string[]} rules[].AllowedOrigin - origins allowed for CORS + * @param {string[]} [rules[].AllowedHeader] - headers allowed in an OPTIONS + * request via the Access-Control-Request-Headers header + * @param {string[]} [rules[].MaxAgeSeconds] - seconds browsers should cache + * OPTIONS response + * @param {string[]} [rules[].ExposeHeader] - headers exposed to applications + * @return {(Error|object)} - return cors object on success; error on failure + */ function _validateCorsXml(rules) { const cors = []; let result; if (rules.length > 100) { - return errorInstances.InvalidRequest - .customizeDescription(customizedErrs.numberRules); + return errorInstances.InvalidRequest.customizeDescription(customizedErrs.numberRules); } for (let i = 0; i < rules.length; i++) { const rule = rules[i]; const corsRule = {}; - result = _validator.validateOriginAndMethodExist(rule.AllowedMethod, - rule.AllowedOrigin); + result = _validator.validateOriginAndMethodExist(rule.AllowedMethod, rule.AllowedOrigin); if (result instanceof Error) { return result; } @@ -240,8 +234,7 @@ function _validateCorsXml(rules) { } corsRule.allowedMethods = rule.AllowedMethod; - result = _validator.validateAllowedOriginsOrHeaders(rule.AllowedOrigin, - 'AllowedOrigin'); + result = _validator.validateAllowedOriginsOrHeaders(rule.AllowedOrigin, 'AllowedOrigin'); if (result instanceof Error) { return result; } @@ -281,12 +274,12 @@ function _validateCorsXml(rules) { } /** parseCorsXml - Parse and validate xml body, returning cors object on success -* @param {string} xml - xml body to parse and validate -* @param {object} log - Werelogs logger -* @param {function} cb - callback to server -* @return {undefined} - calls callback with cors object on success, error on -* failure -*/ + * @param {string} xml - xml body to parse and validate + * @param {object} log - Werelogs logger + * @param {function} cb - callback to server + * @return {undefined} - calls callback with cors object on success, error on + * failure + */ function parseCorsXml(xml, log, cb) { parseString(xml, (err, result) => { if (err) { @@ -298,15 +291,17 @@ function parseCorsXml(xml, log, cb) { return cb(errors.MalformedXML); } - if (!result || !result.CORSConfiguration || + if ( + !result || + !result.CORSConfiguration || !result.CORSConfiguration.CORSRule || - !Array.isArray(result.CORSConfiguration.CORSRule)) { + !Array.isArray(result.CORSConfiguration.CORSRule) + ) { const errMsg = 'Invalid cors configuration xml'; return cb(errorInstances.MalformedXML.customizeDescription(errMsg)); } - const validationRes = - _validateCorsXml(result.CORSConfiguration.CORSRule); + const validationRes = _validateCorsXml(result.CORSConfiguration.CORSRule); if (validationRes instanceof Error) { log.debug('xml validation failed', { error: validationRes, @@ -322,18 +317,14 @@ function parseCorsXml(xml, log, cb) { function convertToXml(arrayRules) { const xml = []; - xml.push('', - ''); + xml.push('', ''); arrayRules.forEach(rule => { xml.push(''); - ['allowedMethods', 'allowedOrigins', 'allowedHeaders', 'exposeHeaders'] - .forEach(key => { + ['allowedMethods', 'allowedOrigins', 'allowedHeaders', 'exposeHeaders'].forEach(key => { if (rule[key]) { - const element = key.charAt(0).toUpperCase() + - key.slice(1, -1); + const element = key.charAt(0).toUpperCase() + key.slice(1, -1); rule[key].forEach(value => { - xml.push(`<${element}>${escapeForXml(value)}` + - ``); + xml.push(`<${element}>${escapeForXml(value)}` + ``); }); } }); diff --git a/lib/api/apiUtils/bucket/bucketDeletion.js b/lib/api/apiUtils/bucket/bucketDeletion.js index 764727e2d2..67d4d1c36b 100644 --- a/lib/api/apiUtils/bucket/bucketDeletion.js +++ b/lib/api/apiUtils/bucket/bucketDeletion.js @@ -5,15 +5,13 @@ const { errors } = require('arsenal'); const abortMultipartUpload = require('../object/abortMultipartUpload'); const { pushMetric } = require('../../../utapi/utilities'); -const { splitter, oldSplitter, mpuBucketPrefix } = - require('../../../../constants'); +const { splitter, oldSplitter, mpuBucketPrefix } = require('../../../../constants'); const metadata = require('../../../metadata/wrapper'); const kms = require('../../../kms/wrapper'); const deleteUserBucketEntry = require('./deleteUserBucketEntry'); function _deleteMPUbucket(destinationBucketName, log, cb) { - const mpuBucketName = - `${mpuBucketPrefix}${destinationBucketName}`; + const mpuBucketName = `${mpuBucketPrefix}${destinationBucketName}`; return metadata.deleteBucket(mpuBucketName, log, err => { // If the mpu bucket does not exist, just move on // TODO: move to `.is` once BKTCLT-9 is done and bumped in Cloudserver @@ -25,23 +23,34 @@ function _deleteMPUbucket(destinationBucketName, log, cb) { } function _deleteOngoingMPUs(authInfo, bucketName, bucketMD, mpus, request, log, cb) { - async.mapLimit(mpus, 1, (mpu, next) => { - const splitterChar = mpu.key.includes(oldSplitter) ? - oldSplitter : splitter; - // `overview${splitter}${objectKey}${splitter}${uploadId} - const [, objectKey, uploadId] = mpu.key.split(splitterChar); - abortMultipartUpload(authInfo, bucketName, objectKey, uploadId, log, - (err, destBucket, partSizeSum) => { - pushMetric('abortMultipartUpload', log, { - authInfo, - canonicalID: bucketMD.getOwner(), - bucket: bucketName, - keys: [objectKey], - byteLength: partSizeSum, - }); - next(err); - }, request); - }, cb); + async.mapLimit( + mpus, + 1, + (mpu, next) => { + const splitterChar = mpu.key.includes(oldSplitter) ? oldSplitter : splitter; + // `overview${splitter}${objectKey}${splitter}${uploadId} + const [, objectKey, uploadId] = mpu.key.split(splitterChar); + abortMultipartUpload( + authInfo, + bucketName, + objectKey, + uploadId, + log, + (err, destBucket, partSizeSum) => { + pushMetric('abortMultipartUpload', log, { + authInfo, + canonicalID: bucketMD.getOwner(), + bucket: bucketName, + keys: [objectKey], + byteLength: partSizeSum, + }); + next(err); + }, + request, + ); + }, + cb, + ); } /** * deleteBucket - Delete bucket from namespace @@ -60,37 +69,36 @@ function deleteBucket(authInfo, bucketMD, bucketName, canonicalID, request, log, assert.strictEqual(typeof bucketName, 'string'); assert.strictEqual(typeof canonicalID, 'string'); - return async.waterfall([ - function checkForObjectsStep(next) { - const params = { maxKeys: 1, listingType: 'DelimiterVersions' }; - // We list all the versions as we want to return BucketNotEmpty - // error if there are any versions or delete markers in the bucket. - // Works for non-versioned buckets as well since listing versions - // includes null (non-versioned) objects in the result. - return metadata.listObject(bucketName, params, log, - (err, list) => { + return async.waterfall( + [ + function checkForObjectsStep(next) { + const params = { maxKeys: 1, listingType: 'DelimiterVersions' }; + // We list all the versions as we want to return BucketNotEmpty + // error if there are any versions or delete markers in the bucket. + // Works for non-versioned buckets as well since listing versions + // includes null (non-versioned) objects in the result. + return metadata.listObject(bucketName, params, log, (err, list) => { if (err) { log.error('error from metadata', { error: err }); return next(err); } - const length = (list.Versions ? list.Versions.length : 0) + + const length = + (list.Versions ? list.Versions.length : 0) + (list.DeleteMarkers ? list.DeleteMarkers.length : 0); log.debug('listing result', { length }); if (length) { - log.debug('bucket delete failed', - { error: errors.BucketNotEmpty }); + log.debug('bucket delete failed', { error: errors.BucketNotEmpty }); return next(errors.BucketNotEmpty); } return next(); }); - }, + }, - function deleteMPUbucketStep(next) { - const MPUBucketName = `${mpuBucketPrefix}${bucketName}`; - // check to see if there are any mpu overview objects (so ignore - // any orphaned part objects) - return metadata.listObject(MPUBucketName, { prefix: 'overview' }, - log, (err, objectsListRes) => { + function deleteMPUbucketStep(next) { + const MPUBucketName = `${mpuBucketPrefix}${bucketName}`; + // check to see if there are any mpu overview objects (so ignore + // any orphaned part objects) + return metadata.listObject(MPUBucketName, { prefix: 'overview' }, log, (err, objectsListRes) => { // If no shadow bucket ever created, no ongoing MPU's, so // continue with deletion if (err?.is?.NoSuchBucket) { @@ -101,58 +109,67 @@ function deleteBucket(authInfo, bucketMD, bucketName, canonicalID, request, log, return next(err); } if (objectsListRes.Contents.length) { - return _deleteOngoingMPUs(authInfo, bucketName, - bucketMD, objectsListRes.Contents, request, log, err => { + return _deleteOngoingMPUs( + authInfo, + bucketName, + bucketMD, + objectsListRes.Contents, + request, + log, + err => { if (err) { return next(err); } log.trace('deleting shadow MPU bucket'); return _deleteMPUbucket(bucketName, log, next); - }); + }, + ); } log.trace('deleting shadow MPU bucket'); return _deleteMPUbucket(bucketName, log, next); }); - }, - function addDeleteFlagStep(next) { - log.trace('adding deleted attribute to bucket attributes'); - // Remove transient flag if any so never have both transient - // and deleted flags. - bucketMD.removeTransientFlag(); - bucketMD.addDeletedFlag(); - return metadata.updateBucket(bucketName, bucketMD, log, next); - }, - function deleteUserBucketEntryStep(next) { - log.trace('deleting bucket name from user bucket'); - return deleteUserBucketEntry(bucketName, canonicalID, log, next); - }, - ], - // eslint-disable-next-line prefer-arrow-callback - function actualDeletionStep(err) { - if (err) { - return cb(err); - } - return metadata.deleteBucket(bucketName, log, err => { - log.trace('deleting bucket from metadata'); + }, + function addDeleteFlagStep(next) { + log.trace('adding deleted attribute to bucket attributes'); + // Remove transient flag if any so never have both transient + // and deleted flags. + bucketMD.removeTransientFlag(); + bucketMD.addDeletedFlag(); + return metadata.updateBucket(bucketName, bucketMD, log, next); + }, + function deleteUserBucketEntryStep(next) { + log.trace('deleting bucket name from user bucket'); + return deleteUserBucketEntry(bucketName, canonicalID, log, next); + }, + ], + // eslint-disable-next-line prefer-arrow-callback + function actualDeletionStep(err) { if (err) { return cb(err); } - const serverSideEncryption = bucketMD.getServerSideEncryption(); - const isScalityManagedEncryptionKey = serverSideEncryption && serverSideEncryption.algorithm === 'AES256'; - const isAccountEncryptionEnabled = bucketMD.isAccountEncryptionEnabled(); + return metadata.deleteBucket(bucketName, log, err => { + log.trace('deleting bucket from metadata'); + if (err) { + return cb(err); + } + const serverSideEncryption = bucketMD.getServerSideEncryption(); + const isScalityManagedEncryptionKey = + serverSideEncryption && serverSideEncryption.algorithm === 'AES256'; + const isAccountEncryptionEnabled = bucketMD.isAccountEncryptionEnabled(); - /** - * If all of the following conditions are met, delete the master encryption key: - * - The encryption key is managed by Scality (not externally managed). - * - The encryption is bucket-specific (to prevent deleting default account encryption key). - */ - if (isScalityManagedEncryptionKey && !isAccountEncryptionEnabled) { - const masterKeyId = serverSideEncryption.masterKeyId; - return kms.destroyBucketKey(masterKeyId, log, cb); - } - return cb(); - }); - }); + /** + * If all of the following conditions are met, delete the master encryption key: + * - The encryption key is managed by Scality (not externally managed). + * - The encryption is bucket-specific (to prevent deleting default account encryption key). + */ + if (isScalityManagedEncryptionKey && !isAccountEncryptionEnabled) { + const masterKeyId = serverSideEncryption.masterKeyId; + return kms.destroyBucketKey(masterKeyId, log, cb); + } + return cb(); + }); + }, + ); } module.exports = deleteBucket; diff --git a/lib/api/apiUtils/bucket/bucketEncryption.js b/lib/api/apiUtils/bucket/bucketEncryption.js index 8916e96f1b..e5a567124c 100644 --- a/lib/api/apiUtils/bucket/bucketEncryption.js +++ b/lib/api/apiUtils/bucket/bucketEncryption.js @@ -11,7 +11,7 @@ const { isScalityKmsArn } = require('arsenal/build/lib/network/KMSInterface'); * @property {string} masterKeyId - Key id for the kms key used to encrypt data keys. * @property {string} configuredMasterKeyId - User configured master key id. * @property {boolean} mandatory - Whether a default encryption policy has been enabled. -*/ + */ /** * @callback ServerSideEncryptionInfo~callback @@ -37,9 +37,7 @@ function parseEncryptionXml(xml, log, cb) { return cb(errors.MalformedXML); } - if (!parsed - || !parsed.ServerSideEncryptionConfiguration - || !parsed.ServerSideEncryptionConfiguration.Rule) { + if (!parsed || !parsed.ServerSideEncryptionConfiguration || !parsed.ServerSideEncryptionConfiguration.Rule) { log.trace('error in sse config, invalid ServerSideEncryptionConfiguration section', { method: 'parseEncryptionXml', }); @@ -48,11 +46,13 @@ function parseEncryptionXml(xml, log, cb) { const { Rule } = parsed.ServerSideEncryptionConfiguration; - if (!Array.isArray(Rule) - || Rule.length > 1 - || !Rule[0] - || !Rule[0].ApplyServerSideEncryptionByDefault - || !Rule[0].ApplyServerSideEncryptionByDefault[0]) { + if ( + !Array.isArray(Rule) || + Rule.length > 1 || + !Rule[0] || + !Rule[0].ApplyServerSideEncryptionByDefault || + !Rule[0].ApplyServerSideEncryptionByDefault[0] + ) { log.trace('error in sse config, invalid ApplyServerSideEncryptionByDefault section', { method: 'parseEncryptionXml', }); @@ -84,8 +84,11 @@ function parseEncryptionXml(xml, log, cb) { log.trace('error in sse config, can not specify KMSMasterKeyID when using AES256', { method: 'parseEncryptionXml', }); - return cb(errorInstances.InvalidArgument.customizeDescription( - 'a KMSMasterKeyID is not applicable if the default sse algorithm is not aws:kms')); + return cb( + errorInstances.InvalidArgument.customizeDescription( + 'a KMSMasterKeyID is not applicable if the default sse algorithm is not aws:kms', + ), + ); } if (!encConfig.KMSMasterKeyID[0] || typeof encConfig.KMSMasterKeyID[0] !== 'string') { @@ -163,15 +166,17 @@ function parseObjectEncryptionHeaders(headers) { if (sseAlgorithm && sseAlgorithm !== 'AES256' && sseAlgorithm !== 'aws:kms') { return { - error: errorInstances.InvalidArgument - .customizeDescription('The encryption method specified is not supported'), + error: errorInstances.InvalidArgument.customizeDescription( + 'The encryption method specified is not supported', + ), }; } if (sseAlgorithm !== 'aws:kms' && configuredMasterKeyId) { return { error: errorInstances.InvalidArgument.customizeDescription( - 'a KMSMasterKeyID is not applicable if the default sse algorithm is not aws:kms'), + 'a KMSMasterKeyID is not applicable if the default sse algorithm is not aws:kms', + ), }; } return { objectSSE: hydrateEncryptionConfig(sseAlgorithm, configuredMasterKeyId) }; @@ -185,17 +190,13 @@ function parseObjectEncryptionHeaders(headers) { * @returns {undefined} */ function createDefaultBucketEncryptionMetadata(bucket, log, cb) { - return kms.bucketLevelEncryption( - bucket, - { algorithm: 'AES256', mandatory: false }, - log, - (error, sseConfig) => { - if (error) { - return cb(error); - } - bucket.setServerSideEncryption(sseConfig); - return metadata.updateBucket(bucket.getName(), bucket, log, err => cb(err, sseConfig)); - }); + return kms.bucketLevelEncryption(bucket, { algorithm: 'AES256', mandatory: false }, log, (error, sseConfig) => { + if (error) { + return cb(error); + } + bucket.setServerSideEncryption(sseConfig); + return metadata.updateBucket(bucket.getName(), bucket, log, err => cb(err, sseConfig)); + }); } /** diff --git a/lib/api/apiUtils/bucket/bucketShield.js b/lib/api/apiUtils/bucket/bucketShield.js index 483a092ee6..fe07b9eada 100644 --- a/lib/api/apiUtils/bucket/bucketShield.js +++ b/lib/api/apiUtils/bucket/bucketShield.js @@ -9,32 +9,32 @@ const constants = require('../../../../constants'); * @return {boolean} true if the bucket should be shielded, false otherwise */ function bucketShield(bucket, requestType) { - const invisiblyDeleteRequests = constants.bucketOwnerActions.concat( - [ - 'bucketGet', - 'bucketHead', - 'bucketGetACL', - 'objectGet', - 'objectGetACL', - 'objectHead', - 'objectPutACL', - 'objectDelete', - ]); - if (invisiblyDeleteRequests.indexOf(requestType) > -1 && - bucket.hasDeletedFlag()) { + const invisiblyDeleteRequests = constants.bucketOwnerActions.concat([ + 'bucketGet', + 'bucketHead', + 'bucketGetACL', + 'objectGet', + 'objectGetACL', + 'objectHead', + 'objectPutACL', + 'objectDelete', + ]); + if (invisiblyDeleteRequests.indexOf(requestType) > -1 && bucket.hasDeletedFlag()) { invisiblyDelete(bucket.getName(), bucket.getOwner()); return true; } - // If request is initiateMultipartUpload (requestType objectPut), - // objectPut, bucketPutACL or bucketDelete, proceed with request. - // Otherwise return an error to the client - if ((bucket.hasDeletedFlag() || bucket.hasTransientFlag()) && - (requestType !== 'objectPut' && + // If request is initiateMultipartUpload (requestType objectPut), + // objectPut, bucketPutACL or bucketDelete, proceed with request. + // Otherwise return an error to the client + if ( + (bucket.hasDeletedFlag() || bucket.hasTransientFlag()) && + requestType !== 'objectPut' && requestType !== 'initiateMultipartUpload' && requestType !== 'objectPutPart' && requestType !== 'completeMultipartUpload' && requestType !== 'bucketPutACL' && - requestType !== 'bucketDelete')) { + requestType !== 'bucketDelete' + ) { return true; } return false; diff --git a/lib/api/apiUtils/bucket/bucketWebsite.js b/lib/api/apiUtils/bucket/bucketWebsite.js index e723e2387a..e909091bef 100644 --- a/lib/api/apiUtils/bucket/bucketWebsite.js +++ b/lib/api/apiUtils/bucket/bucketWebsite.js @@ -2,8 +2,7 @@ const { parseString } = require('xml2js'); const { errors, errorInstances, s3middleware } = require('arsenal'); const escapeForXml = s3middleware.escapeForXml; -const { WebsiteConfiguration } = - require('arsenal').models.WebsiteConfiguration; +const { WebsiteConfiguration } = require('arsenal').models.WebsiteConfiguration; /* Format of xml request: @@ -30,31 +29,29 @@ const { WebsiteConfiguration } = */ - // Key names of redirect object values to check if are valid strings -const redirectValuesToCheck = ['HostName', 'ReplaceKeyPrefixWith', - 'ReplaceKeyWith']; +const redirectValuesToCheck = ['HostName', 'ReplaceKeyPrefixWith', 'ReplaceKeyWith']; /** Helper function for validating format of parsed xml element -* @param {array} elem - element to check -* @return {boolean} true / false - elem meets expected format -*/ + * @param {array} elem - element to check + * @return {boolean} true / false - elem meets expected format + */ function _isValidElem(elem) { - return (Array.isArray(elem) && elem.length === 1); + return Array.isArray(elem) && elem.length === 1; } /** Check if parsed xml element contains a specified child element -* @param {array} parent - represents xml element to check for child element -* @param {(string|string[])} requiredElem - name of child element(s) -* @param {object} [options] - specify additional options -* @param {boolean} [isList] - indicates if parent is list of children elements, -* used only in conjunction a singular requiredElem argument -* @param {boolean} [checkForAll] - return true only if parent element contains -* all children elements specified in requiredElem; by default, returns true if -* parent element contains at least one -* @param {boolean} [validateParent] - validate format of parent element -* @return {boolean} true / false - if parsed xml element contains child -*/ + * @param {array} parent - represents xml element to check for child element + * @param {(string|string[])} requiredElem - name of child element(s) + * @param {object} [options] - specify additional options + * @param {boolean} [isList] - indicates if parent is list of children elements, + * used only in conjunction a singular requiredElem argument + * @param {boolean} [checkForAll] - return true only if parent element contains + * all children elements specified in requiredElem; by default, returns true if + * parent element contains at least one + * @param {boolean} [validateParent] - validate format of parent element + * @return {boolean} true / false - if parsed xml element contains child + */ function xmlContainsElem(parent, requiredElem, options) { // Non-top level xml is parsed into object in the following manner. @@ -75,8 +72,7 @@ function xmlContainsElem(parent, requiredElem, options) { const checkForAll = options ? options.checkForAll : false; // true by default, validateParent only designated as false when // parent was validated in previous check - const validateParent = (options && options.validateParent !== undefined) ? - options.validateParent : true; + const validateParent = options && options.validateParent !== undefined ? options.validateParent : true; if (validateParent && !_isValidElem(parent)) { return false; @@ -88,8 +84,7 @@ function xmlContainsElem(parent, requiredElem, options) { return requiredElem.some(elem => _isValidElem(parent[0][elem])); } if (isList) { - if (!Array.isArray(parent[0][requiredElem]) || - parent[0][requiredElem].length === 0) { + if (!Array.isArray(parent[0][requiredElem]) || parent[0][requiredElem].length === 0) { return false; } } else { @@ -99,7 +94,6 @@ function xmlContainsElem(parent, requiredElem, options) { return true; } - /** Validate XML, returning an error if any part is not valid * @param {object} parsingResult - object parsed from xml to be validated * @param {object[]} parsingResult.IndexDocument - @@ -146,22 +140,19 @@ function _validateWebsiteConfigXml(parsingResult) { let errMsg; function _isValidString(value) { - return (typeof value === 'string' && value !== ''); + return typeof value === 'string' && value !== ''; } if (!parsingResult.IndexDocument && !parsingResult.RedirectAllRequestsTo) { - errMsg = 'Value for IndexDocument Suffix must be provided if ' + - 'RedirectAllRequestsTo is empty'; + errMsg = 'Value for IndexDocument Suffix must be provided if ' + 'RedirectAllRequestsTo is empty'; return errorInstances.InvalidArgument.customizeDescription(errMsg); } if (parsingResult.RedirectAllRequestsTo) { const parent = parsingResult.RedirectAllRequestsTo; const redirectAllObj = {}; - if (parsingResult.IndexDocument || parsingResult.ErrorDocument || - parsingResult.RoutingRules) { - errMsg = 'RedirectAllRequestsTo cannot be provided in ' + - 'conjunction with other Routing Rules.'; + if (parsingResult.IndexDocument || parsingResult.ErrorDocument || parsingResult.RoutingRules) { + errMsg = 'RedirectAllRequestsTo cannot be provided in ' + 'conjunction with other Routing Rules.'; return errorInstances.InvalidRequest.customizeDescription(errMsg); } if (!xmlContainsElem(parent, 'HostName')) { @@ -174,10 +165,10 @@ function _validateWebsiteConfigXml(parsingResult) { } redirectAllObj.hostName = parent[0].HostName[0]; if (xmlContainsElem(parent, 'Protocol', { validateParent: false })) { - if (parent[0].Protocol[0] !== 'http' && - parent[0].Protocol[0] !== 'https') { - errMsg = 'Invalid protocol, protocol can be http or https. ' + - 'If not defined, the protocol will be selected automatically.'; + if (parent[0].Protocol[0] !== 'http' && parent[0].Protocol[0] !== 'https') { + errMsg = + 'Invalid protocol, protocol can be http or https. ' + + 'If not defined, the protocol will be selected automatically.'; return errorInstances.InvalidRequest.customizeDescription(errMsg); } redirectAllObj.protocol = parent[0].Protocol[0]; @@ -190,8 +181,7 @@ function _validateWebsiteConfigXml(parsingResult) { if (!xmlContainsElem(parent, 'Suffix')) { errMsg = 'IndexDocument is not well-formed'; return errorInstances.MalformedXML.customizeDescription(errMsg); - } else if (!_isValidString(parent[0].Suffix[0]) - || parent[0].Suffix[0].indexOf('/') !== -1) { + } else if (!_isValidString(parent[0].Suffix[0]) || parent[0].Suffix[0].indexOf('/') !== -1) { errMsg = 'IndexDocument Suffix is not well-formed'; return errorInstances.InvalidArgument.customizeDescription(errMsg); } @@ -221,8 +211,7 @@ function _validateWebsiteConfigXml(parsingResult) { const rule = parent[0].RoutingRule[i]; const ruleObj = { redirect: {} }; if (!_isValidElem(rule.Redirect)) { - errMsg = 'RoutingRule requires Redirect, which is ' + - 'missing or not well-formed'; + errMsg = 'RoutingRule requires Redirect, which is ' + 'missing or not well-formed'; return errorInstances.MalformedXML.customizeDescription(errMsg); } // Looks like AWS doesn't actually make this check, but AWS @@ -231,28 +220,35 @@ function _validateWebsiteConfigXml(parsingResult) { // elements to know how to implement a redirect for a rule. // http://docs.aws.amazon.com/AmazonS3/latest/API/ // RESTBucketPUTwebsite.html - if (!xmlContainsElem(rule.Redirect, ['Protocol', 'HostName', - 'ReplaceKeyPrefixWith', 'ReplaceKeyWith', 'HttpRedirectCode'], - { validateParent: false })) { - errMsg = 'Redirect must contain at least one of ' + - 'following: Protocol, HostName, ReplaceKeyPrefixWith, ' + - 'ReplaceKeyWith, or HttpRedirectCode element'; + if ( + !xmlContainsElem( + rule.Redirect, + ['Protocol', 'HostName', 'ReplaceKeyPrefixWith', 'ReplaceKeyWith', 'HttpRedirectCode'], + { validateParent: false }, + ) + ) { + errMsg = + 'Redirect must contain at least one of ' + + 'following: Protocol, HostName, ReplaceKeyPrefixWith, ' + + 'ReplaceKeyWith, or HttpRedirectCode element'; return errorInstances.MalformedXML.customizeDescription(errMsg); } if (rule.Redirect[0].Protocol) { - if (!_isValidElem(rule.Redirect[0].Protocol) || - (rule.Redirect[0].Protocol[0] !== 'http' && - rule.Redirect[0].Protocol[0] !== 'https')) { - errMsg = 'Invalid protocol, protocol can be http or ' + - 'https. If not defined, the protocol will be selected ' + - 'automatically.'; + if ( + !_isValidElem(rule.Redirect[0].Protocol) || + (rule.Redirect[0].Protocol[0] !== 'http' && rule.Redirect[0].Protocol[0] !== 'https') + ) { + errMsg = + 'Invalid protocol, protocol can be http or ' + + 'https. If not defined, the protocol will be selected ' + + 'automatically.'; return errorInstances.InvalidRequest.customizeDescription(errMsg); } ruleObj.redirect.protocol = rule.Redirect[0].Protocol[0]; } if (rule.Redirect[0].HttpRedirectCode) { - errMsg = 'The provided HTTP redirect code is not valid. ' + - 'It should be a string containing a number.'; + errMsg = + 'The provided HTTP redirect code is not valid. ' + 'It should be a string containing a number.'; if (!_isValidElem(rule.Redirect[0].HttpRedirectCode)) { return errorInstances.MalformedXML.customizeDescription(errMsg); } @@ -261,8 +257,8 @@ function _validateWebsiteConfigXml(parsingResult) { return errorInstances.MalformedXML.customizeDescription(errMsg); } if (!(code > 300 && code < 400)) { - errMsg = `The provided HTTP redirect code (${code}) is ` + - 'not valid. Valid codes are 3XX except 300'; + errMsg = + `The provided HTTP redirect code (${code}) is ` + 'not valid. Valid codes are 3XX except 300'; return errorInstances.InvalidRequest.customizeDescription(errMsg); } ruleObj.redirect.httpRedirectCode = code; @@ -273,57 +269,51 @@ function _validateWebsiteConfigXml(parsingResult) { if (elem) { if (!_isValidElem(elem) || !_isValidString(elem[0])) { errMsg = `Redirect ${elem} is not well-formed`; - return errorInstances.InvalidArgument - .customizeDescription(errMsg); + return errorInstances.InvalidArgument.customizeDescription(errMsg); } - ruleObj.redirect[`${elemName.charAt(0).toLowerCase()}` + - `${elemName.slice(1)}`] = elem[0]; + ruleObj.redirect[`${elemName.charAt(0).toLowerCase()}` + `${elemName.slice(1)}`] = elem[0]; } } - if (xmlContainsElem( - rule.Redirect, - ['ReplaceKeyPrefixWith', 'ReplaceKeyWith'], - { validateParent: false, checkForAll: true })) { - errMsg = 'Redirect must not contain both ReplaceKeyWith ' + - 'and ReplaceKeyPrefixWith'; + if ( + xmlContainsElem(rule.Redirect, ['ReplaceKeyPrefixWith', 'ReplaceKeyWith'], { + validateParent: false, + checkForAll: true, + }) + ) { + errMsg = 'Redirect must not contain both ReplaceKeyWith ' + 'and ReplaceKeyPrefixWith'; return errorInstances.InvalidRequest.customizeDescription(errMsg); } if (Array.isArray(rule.Condition) && rule.Condition.length === 1) { ruleObj.condition = {}; - if (!xmlContainsElem(rule.Condition, ['KeyPrefixEquals', - 'HttpErrorCodeReturnedEquals'])) { - errMsg = 'Condition is not well-formed. ' + - 'Condition should contain valid KeyPrefixEquals or ' + - 'HttpErrorCodeReturnEquals element.'; + if (!xmlContainsElem(rule.Condition, ['KeyPrefixEquals', 'HttpErrorCodeReturnedEquals'])) { + errMsg = + 'Condition is not well-formed. ' + + 'Condition should contain valid KeyPrefixEquals or ' + + 'HttpErrorCodeReturnEquals element.'; return errorInstances.InvalidRequest.customizeDescription(errMsg); } if (rule.Condition[0].KeyPrefixEquals) { const keyPrefixEquals = rule.Condition[0].KeyPrefixEquals; - if (!_isValidElem(keyPrefixEquals) || - !_isValidString(keyPrefixEquals[0])) { + if (!_isValidElem(keyPrefixEquals) || !_isValidString(keyPrefixEquals[0])) { errMsg = 'Condition KeyPrefixEquals is not well-formed'; - return errorInstances.InvalidArgument - .customizeDescription(errMsg); + return errorInstances.InvalidArgument.customizeDescription(errMsg); } ruleObj.condition.keyPrefixEquals = keyPrefixEquals[0]; } if (rule.Condition[0].HttpErrorCodeReturnedEquals) { - errMsg = 'The provided HTTP error code is not valid. ' + - 'It should be a string containing a number.'; - if (!_isValidElem(rule.Condition[0] - .HttpErrorCodeReturnedEquals)) { + errMsg = + 'The provided HTTP error code is not valid. ' + 'It should be a string containing a number.'; + if (!_isValidElem(rule.Condition[0].HttpErrorCodeReturnedEquals)) { return errorInstances.MalformedXML.customizeDescription(errMsg); } - const code = parseInt(rule.Condition[0] - .HttpErrorCodeReturnedEquals[0], 10); + const code = parseInt(rule.Condition[0].HttpErrorCodeReturnedEquals[0], 10); if (Number.isNaN(code)) { return errorInstances.MalformedXML.customizeDescription(errMsg); } if (!(code > 399 && code < 600)) { - errMsg = `The provided HTTP error code (${code}) is ` + - 'not valid. Valid codes are 4XX or 5XX.'; - return errorInstances.InvalidRequest - .customizeDescription(errMsg); + errMsg = + `The provided HTTP error code (${code}) is ` + 'not valid. Valid codes are 4XX or 5XX.'; + return errorInstances.InvalidRequest.customizeDescription(errMsg); } ruleObj.condition.httpErrorCodeReturnedEquals = code; } @@ -350,8 +340,7 @@ function parseWebsiteConfigXml(xml, log, cb) { return cb(errorInstances.MalformedXML.customizeDescription(errMsg)); } - const validationRes = - _validateWebsiteConfigXml(result.WebsiteConfiguration); + const validationRes = _validateWebsiteConfigXml(result.WebsiteConfiguration); if (validationRes instanceof Error) { log.debug('xml validation failed', { error: validationRes, @@ -375,35 +364,27 @@ function convertToXml(config) { function _pushChildren(obj) { Object.keys(obj).forEach(element => { - const xmlElem = `${element.charAt(0).toUpperCase()}` + - `${element.slice(1)}`; + const xmlElem = `${element.charAt(0).toUpperCase()}` + `${element.slice(1)}`; xml.push(`<${xmlElem}>${escapeForXml(obj[element])}`); }); } - xml.push('', - ''); + xml.push( + '', + '', + ); if (indexDocument) { - xml.push('', - `${escapeForXml(indexDocument)}`, - ''); + xml.push('', `${escapeForXml(indexDocument)}`, ''); } if (errorDocument) { - xml.push('', - `${escapeForXml(errorDocument)}`, - ''); + xml.push('', `${escapeForXml(errorDocument)}`, ''); } if (redirectAllRequestsTo) { xml.push(''); if (redirectAllRequestsTo.hostName) { - xml.push('', - `${escapeForXml(redirectAllRequestsTo.hostName)}`, - ''); + xml.push('', `${escapeForXml(redirectAllRequestsTo.hostName)}`, ''); } if (redirectAllRequestsTo.protocol) { - xml.push('', - `${redirectAllRequestsTo.protocol}`, - ''); + xml.push('', `${redirectAllRequestsTo.protocol}`, ''); } xml.push(''); } diff --git a/lib/api/apiUtils/bucket/checkPreferredLocations.js b/lib/api/apiUtils/bucket/checkPreferredLocations.js index c717c079c5..1c384479f0 100644 --- a/lib/api/apiUtils/bucket/checkPreferredLocations.js +++ b/lib/api/apiUtils/bucket/checkPreferredLocations.js @@ -2,10 +2,10 @@ const { errorInstances } = require('arsenal'); function checkPreferredLocations(location, locationConstraints, log) { const retError = loc => { - const errMsg = 'value of the location you are attempting to set - ' + - `${loc} - is not listed in the locationConstraint config`; - log.trace(`locationConstraint is invalid - ${errMsg}`, - { locationConstraint: loc }); + const errMsg = + 'value of the location you are attempting to set - ' + + `${loc} - is not listed in the locationConstraint config`; + log.trace(`locationConstraint is invalid - ${errMsg}`, { locationConstraint: loc }); return errorInstances.InvalidLocationConstraint.customizeDescription(errMsg); }; if (typeof location === 'string' && !locationConstraints[location]) { diff --git a/lib/api/apiUtils/bucket/createKeyForUserBucket.js b/lib/api/apiUtils/bucket/createKeyForUserBucket.js index 36f23c3706..f4d46e8042 100644 --- a/lib/api/apiUtils/bucket/createKeyForUserBucket.js +++ b/lib/api/apiUtils/bucket/createKeyForUserBucket.js @@ -1,5 +1,4 @@ -function createKeyForUserBucket(canonicalID, - splitter, bucketName) { +function createKeyForUserBucket(canonicalID, splitter, bucketName) { return `${canonicalID}${splitter}${bucketName}`; } diff --git a/lib/api/apiUtils/bucket/deleteUserBucketEntry.js b/lib/api/apiUtils/bucket/deleteUserBucketEntry.js index ff1846e963..8257a67785 100644 --- a/lib/api/apiUtils/bucket/deleteUserBucketEntry.js +++ b/lib/api/apiUtils/bucket/deleteUserBucketEntry.js @@ -1,42 +1,35 @@ const createKeyForUserBucket = require('./createKeyForUserBucket'); -const { usersBucket, oldUsersBucket, splitter, oldSplitter } = - require('../../../../constants'); +const { usersBucket, oldUsersBucket, splitter, oldSplitter } = require('../../../../constants'); const metadata = require('../../../metadata/wrapper'); function deleteUserBucketEntry(bucketName, canonicalID, log, cb) { - log.trace('deleting bucket name from users bucket', { method: - '_deleteUserBucketEntry' }); - const keyForUserBucket = createKeyForUserBucket(canonicalID, splitter, - bucketName); + log.trace('deleting bucket name from users bucket', { method: '_deleteUserBucketEntry' }); + const keyForUserBucket = createKeyForUserBucket(canonicalID, splitter, bucketName); metadata.deleteObjectMD(usersBucket, keyForUserBucket, {}, log, error => { // If the object representing the bucket is not in the // users bucket just continue if (error?.is.NoSuchKey) { return cb(null); - // BACKWARDS COMPATIBILITY: Remove this once no longer - // have old user bucket format + // BACKWARDS COMPATIBILITY: Remove this once no longer + // have old user bucket format } else if (error?.is.NoSuchBucket) { - const keyForUserBucket2 = createKeyForUserBucket(canonicalID, - oldSplitter, bucketName); - return metadata.deleteObjectMD(oldUsersBucket, keyForUserBucket2, - {}, log, error => { - // TODO: move to `.is` once BKTCLT-9 is done and bumped in Cloudserver - if (error && !error.NoSuchKey) { - log.error('from metadata while deleting user bucket', - { error }); - return cb(error); - } - log.trace('deleted bucket from user bucket', - { method: '_deleteUserBucketEntry' }); - return cb(null); - }); + const keyForUserBucket2 = createKeyForUserBucket(canonicalID, oldSplitter, bucketName); + return metadata.deleteObjectMD(oldUsersBucket, keyForUserBucket2, {}, log, error => { + // TODO: move to `.is` once BKTCLT-9 is done and bumped in Cloudserver + if (error && !error.NoSuchKey) { + log.error('from metadata while deleting user bucket', { error }); + return cb(error); + } + log.trace('deleted bucket from user bucket', { method: '_deleteUserBucketEntry' }); + return cb(null); + }); } else if (error) { - log.error('from metadata while deleting user bucket', { error, - method: '_deleteUserBucketEntry' }); + log.error('from metadata while deleting user bucket', { error, method: '_deleteUserBucketEntry' }); return cb(error); } log.trace('deleted bucket from user bucket', { - method: '_deleteUserBucketEntry' }); + method: '_deleteUserBucketEntry', + }); return cb(null); }); } diff --git a/lib/api/apiUtils/bucket/getNotificationConfiguration.js b/lib/api/apiUtils/bucket/getNotificationConfiguration.js index 1f7151235f..2d9ba03d92 100644 --- a/lib/api/apiUtils/bucket/getNotificationConfiguration.js +++ b/lib/api/apiUtils/bucket/getNotificationConfiguration.js @@ -10,8 +10,11 @@ function getNotificationConfiguration(parsedXml) { return notifConfig; } if (!config.bucketNotificationDestinations) { - return { error: errorInstances.InvalidArgument.customizeDescription( - 'Unable to validate the following destination configurations') }; + return { + error: errorInstances.InvalidArgument.customizeDescription( + 'Unable to validate the following destination configurations', + ), + }; } const targets = new Set(config.bucketNotificationDestinations.map(t => t.resource)); const notifConfigTargets = notifConfig.queueConfig.map(t => t.queueArn.split(':')[5]); diff --git a/lib/api/apiUtils/bucket/getReplicationConfiguration.js b/lib/api/apiUtils/bucket/getReplicationConfiguration.js index 1f0e5231d7..9f6bf2c34c 100644 --- a/lib/api/apiUtils/bucket/getReplicationConfiguration.js +++ b/lib/api/apiUtils/bucket/getReplicationConfiguration.js @@ -1,7 +1,6 @@ const config = require('../../../Config').config; const parseXML = require('../../../utilities/parseXML'); -const ReplicationConfiguration = - require('arsenal').models.ReplicationConfiguration; +const ReplicationConfiguration = require('arsenal').models.ReplicationConfiguration; // Handle the steps for returning a valid replication configuration object. function getReplicationConfiguration(xml, log, cb) { diff --git a/lib/api/apiUtils/bucket/invisiblyDelete.js b/lib/api/apiUtils/bucket/invisiblyDelete.js index 5ccf5cd329..2e39cc39a4 100644 --- a/lib/api/apiUtils/bucket/invisiblyDelete.js +++ b/lib/api/apiUtils/bucket/invisiblyDelete.js @@ -15,20 +15,17 @@ function invisiblyDelete(bucketName, canonicalID) { log.trace('deleting bucket with deleted flag invisibly', { bucketName }); return deleteUserBucketEntry(bucketName, canonicalID, log, err => { if (err) { - log.error('error invisibly deleting bucket name from user bucket', - { error: err }); + log.error('error invisibly deleting bucket name from user bucket', { error: err }); return log.end(); } log.trace('deleted bucket name from user bucket'); return metadata.deleteBucket(bucketName, log, error => { - log.trace('deleting bucket from metadata', - { method: 'invisiblyDelete' }); + log.trace('deleting bucket from metadata', { method: 'invisiblyDelete' }); if (error) { log.error('error deleting bucket from metadata', { error }); return log.end(); } - log.trace('invisible deletion of bucket succeeded', - { method: 'invisiblyDelete' }); + log.trace('invisible deletion of bucket succeeded', { method: 'invisiblyDelete' }); return log.end(); }); }); diff --git a/lib/api/apiUtils/bucket/parseWhere.js b/lib/api/apiUtils/bucket/parseWhere.js index 4275f9e1cb..83deccfced 100644 --- a/lib/api/apiUtils/bucket/parseWhere.js +++ b/lib/api/apiUtils/bucket/parseWhere.js @@ -38,7 +38,7 @@ const exprMapper = { '<': '$lt', '>=': '$gte', '<=': '$lte', - 'LIKE': '$regex', + LIKE: '$regex', }; /* @@ -53,18 +53,12 @@ function parseWhere(root) { const e1 = parseWhere(root[operator][0]); const e2 = parseWhere(root[operator][1]); - return { '$and' : [ - e1, - e2, - ] }; + return { $and: [e1, e2] }; } else if (operator === 'OR') { const e1 = parseWhere(root[operator][0]); const e2 = parseWhere(root[operator][1]); - return { '$or' : [ - e1, - e2, - ] }; + return { $or: [e1, e2] }; } const field = root[operator][0]; const value = root[operator][1]; diff --git a/lib/api/apiUtils/bucket/updateEncryption.js b/lib/api/apiUtils/bucket/updateEncryption.js index 5db3152423..515b360dd2 100644 --- a/lib/api/apiUtils/bucket/updateEncryption.js +++ b/lib/api/apiUtils/bucket/updateEncryption.js @@ -87,8 +87,7 @@ function updateObjectEncryption(bucket, objMD, objectKey, log, keyArnPrefix, opt if (opts.skipObjectUpdate) { return cb(null, bucket, objMD); } - return metadata.putObjectMD(bucket.getName(), objectKey, objMD, params, - log, err => cb(err, bucket, objMD)); + return metadata.putObjectMD(bucket.getName(), objectKey, objMD, params, log, err => cb(err, bucket, objMD)); } /** diff --git a/lib/api/apiUtils/bucket/validateReplicationConfig.js b/lib/api/apiUtils/bucket/validateReplicationConfig.js index e0e64df2ca..36807ae14b 100644 --- a/lib/api/apiUtils/bucket/validateReplicationConfig.js +++ b/lib/api/apiUtils/bucket/validateReplicationConfig.js @@ -24,8 +24,7 @@ function validateReplicationConfig(repConfig, bucket) { return true; } const storageClasses = rule.storageClass.split(','); - return storageClasses.some( - site => site.endsWith(':preferred_read')); + return storageClasses.some(site => site.endsWith(':preferred_read')); }); } diff --git a/lib/api/apiUtils/bucket/validateSearch.js b/lib/api/apiUtils/bucket/validateSearch.js index 9a2e33f94c..32d33ed1e3 100644 --- a/lib/api/apiUtils/bucket/validateSearch.js +++ b/lib/api/apiUtils/bucket/validateSearch.js @@ -20,7 +20,7 @@ const sqlConfig = { ], tokenizer: { shouldTokenize: ['(', ')', '=', '!=', '<', '>', '<=', '>=', '<>'], - shouldMatch: ['"', '\'', '`'], + shouldMatch: ['"', "'", '`'], shouldDelimitBy: [' ', '\n', '\r', '\t'], }, }; @@ -39,10 +39,12 @@ function _validateTree(whereClause, possibleAttributes) { _searchTree(node[operator][1]); } else { const field = node[operator][0]; - if (!field.startsWith('tags.') && + if ( + !field.startsWith('tags.') && !possibleAttributes[field] && !field.startsWith('replicationInfo.') && - !field.startsWith('x-amz-meta-')) { + !field.startsWith('x-amz-meta-') + ) { invalidAttribute = field; } } @@ -68,15 +70,14 @@ function validateSearchParams(searchParams) { // allow using 'replicationStatus' as search param to increase // ease of use, pending metadata search rework // eslint-disable-next-line no-param-reassign - searchParams = searchParams.replace( - 'replication-status', 'replicationInfo.status'); + searchParams = searchParams.replace('replication-status', 'replicationInfo.status'); ast = parser.parse(searchParams); } catch (e) { if (e) { return { - error: errorInstances.InvalidArgument - .customizeDescription('Invalid sql where clause ' + - 'sent as search query'), + error: errorInstances.InvalidArgument.customizeDescription( + 'Invalid sql where clause ' + 'sent as search query', + ), }; } } @@ -84,9 +85,10 @@ function validateSearchParams(searchParams) { const invalidAttribute = _validateTree(ast, possibleAttributes); if (invalidAttribute) { return { - error: errorInstances.InvalidArgument - .customizeDescription('Search param ' + - `contains unknown attribute: ${invalidAttribute}`) }; + error: errorInstances.InvalidArgument.customizeDescription( + 'Search param ' + `contains unknown attribute: ${invalidAttribute}`, + ), + }; } return { ast, diff --git a/lib/api/apiUtils/integrity/validateChecksums.js b/lib/api/apiUtils/integrity/validateChecksums.js index 36864604cf..2d03b3a9c7 100644 --- a/lib/api/apiUtils/integrity/validateChecksums.js +++ b/lib/api/apiUtils/integrity/validateChecksums.js @@ -37,22 +37,22 @@ function defaultValidationFunc(request, body, log) { } const methodValidationFunc = Object.freeze({ - 'bucketPutACL': defaultValidationFunc, - 'bucketPutCors': defaultValidationFunc, - 'bucketPutEncryption': defaultValidationFunc, - 'bucketPutLifecycle': defaultValidationFunc, - 'bucketPutNotification': defaultValidationFunc, - 'bucketPutObjectLock': defaultValidationFunc, - 'bucketPutPolicy': defaultValidationFunc, - 'bucketPutReplication': defaultValidationFunc, - 'bucketPutVersioning': defaultValidationFunc, - 'bucketPutWebsite': defaultValidationFunc, + bucketPutACL: defaultValidationFunc, + bucketPutCors: defaultValidationFunc, + bucketPutEncryption: defaultValidationFunc, + bucketPutLifecycle: defaultValidationFunc, + bucketPutNotification: defaultValidationFunc, + bucketPutObjectLock: defaultValidationFunc, + bucketPutPolicy: defaultValidationFunc, + bucketPutReplication: defaultValidationFunc, + bucketPutVersioning: defaultValidationFunc, + bucketPutWebsite: defaultValidationFunc, // TODO: DeleteObjects requires a checksum. Should return an error if ChecksumError.MissingChecksum. - 'multiObjectDelete': defaultValidationFunc, - 'objectPutACL': defaultValidationFunc, - 'objectPutLegalHold': defaultValidationFunc, - 'objectPutTagging': defaultValidationFunc, - 'objectPutRetention': defaultValidationFunc, + multiObjectDelete: defaultValidationFunc, + objectPutACL: defaultValidationFunc, + objectPutLegalHold: defaultValidationFunc, + objectPutTagging: defaultValidationFunc, + objectPutRetention: defaultValidationFunc, }); /** diff --git a/lib/api/apiUtils/object/applyZenkoUserMD.js b/lib/api/apiUtils/object/applyZenkoUserMD.js index 928b56b9f3..48e661804d 100644 --- a/lib/api/apiUtils/object/applyZenkoUserMD.js +++ b/lib/api/apiUtils/object/applyZenkoUserMD.js @@ -9,8 +9,7 @@ const _config = require('../../../Config').config; * @return {undefined} */ function applyZenkoUserMD(metaHeaders) { - if (process.env.REMOTE_MANAGEMENT_DISABLE === '0' && - !metaHeaders[zenkoIDHeader]) { + if (process.env.REMOTE_MANAGEMENT_DISABLE === '0' && !metaHeaders[zenkoIDHeader]) { // eslint-disable-next-line no-param-reassign metaHeaders[zenkoIDHeader] = _config.getPublicInstanceId(); } diff --git a/lib/api/apiUtils/object/checkHttpHeadersSize.js b/lib/api/apiUtils/object/checkHttpHeadersSize.js index 01cf136d7d..9d653c25e7 100644 --- a/lib/api/apiUtils/object/checkHttpHeadersSize.js +++ b/lib/api/apiUtils/object/checkHttpHeadersSize.js @@ -10,8 +10,7 @@ function checkHttpHeadersSize(requestHeaders) { let httpHeadersSize = 0; Object.keys(requestHeaders).forEach(header => { - httpHeadersSize += Buffer.byteLength(header, 'utf8') + - Buffer.byteLength(requestHeaders[header], 'utf8'); + httpHeadersSize += Buffer.byteLength(header, 'utf8') + Buffer.byteLength(requestHeaders[header], 'utf8'); }); if (httpHeadersSize > maxHttpHeadersSize) { diff --git a/lib/api/apiUtils/object/checkReadLocation.js b/lib/api/apiUtils/object/checkReadLocation.js index ce21fee04f..13713385de 100644 --- a/lib/api/apiUtils/object/checkReadLocation.js +++ b/lib/api/apiUtils/object/checkReadLocation.js @@ -11,10 +11,8 @@ function checkReadLocation(config, locationName, objectKey, bucketName) { const readLocation = config.getLocationConstraint(locationName); if (readLocation) { - const bucketMatch = readLocation.details && - readLocation.details.bucketMatch; - const backendKey = bucketMatch ? objectKey : - `${bucketName}/${objectKey}`; + const bucketMatch = readLocation.details && readLocation.details.bucketMatch; + const backendKey = bucketMatch ? objectKey : `${bucketName}/${objectKey}`; return { location: locationName, key: backendKey, diff --git a/lib/api/apiUtils/object/checkUserMetadataSize.js b/lib/api/apiUtils/object/checkUserMetadataSize.js index eb8ac11a9c..28e83e55a7 100644 --- a/lib/api/apiUtils/object/checkUserMetadataSize.js +++ b/lib/api/apiUtils/object/checkUserMetadataSize.js @@ -1,5 +1,4 @@ -const { maximumMetaHeadersSize, - invalidObjectUserMetadataHeader } = require('../../../../constants'); +const { maximumMetaHeadersSize, invalidObjectUserMetadataHeader } = require('../../../../constants'); /** * Checks the size of the user metadata in the object metadata and removes @@ -13,8 +12,7 @@ const { maximumMetaHeadersSize, function checkUserMetadataSize(responseMetadata) { let userMetadataSize = 0; // collect the user metadata keys from the object metadata - const userMetadataHeaders = Object.keys(responseMetadata) - .filter(key => key.startsWith('x-amz-meta-')); + const userMetadataHeaders = Object.keys(responseMetadata).filter(key => key.startsWith('x-amz-meta-')); // compute the size of all user metadata key and its value userMetadataHeaders.forEach(header => { userMetadataSize += header.length + responseMetadata[header].length; diff --git a/lib/api/apiUtils/object/coldStorage.js b/lib/api/apiUtils/object/coldStorage.js index dd33022c99..ab7e8f22b2 100644 --- a/lib/api/apiUtils/object/coldStorage.js +++ b/lib/api/apiUtils/object/coldStorage.js @@ -17,9 +17,7 @@ const { scaledMsPerDay } = config.getTimeOptions(); * @returns {string|undefined} x-amz-restore */ function getAmzRestoreResHeader(objMD) { - if (objMD.archive && - objMD.archive.restoreRequestedAt && - !objMD.archive.restoreCompletedAt) { + if (objMD.archive && objMD.archive.restoreRequestedAt && !objMD.archive.restoreCompletedAt) { // Avoid race condition by relying on the `archive` MD of the object // and return the right header after a RESTORE request. // eslint-disable-next-line @@ -83,11 +81,10 @@ function _validateStartRestore(objectMD, log) { if (new Date(objectMD.archive?.restoreWillExpireAt) < new Date(Date.now())) { // return InvalidObjectState error if the restored object is expired // but restore info md of this object has not yet been cleared - log.debug('The restored object already expired.', - { - archive: objectMD.archive, - method: '_validateStartRestore', - }); + log.debug('The restored object already expired.', { + archive: objectMD.archive, + method: '_validateStartRestore', + }); return errors.InvalidObjectState; } @@ -100,21 +97,19 @@ function _validateStartRestore(objectMD, log) { if (!isLocationCold) { // return InvalidObjectState error if the object is not in cold storage, // not in cold storage means either location cold flag not exists or cold flag is explicit false - log.debug('The bucket of the object is not in a cold storage location.', - { - isLocationCold, - method: '_validateStartRestore', - }); + log.debug('The bucket of the object is not in a cold storage location.', { + isLocationCold, + method: '_validateStartRestore', + }); return errors.InvalidObjectState; } if (objectMD.archive?.restoreRequestedAt) { // return RestoreAlreadyInProgress error if the object is currently being restored // check if archive.restoreRequestAt exists and archive.restoreCompletedAt not yet exists - log.debug('The object is currently being restored.', - { - archive: objectMD.archive, - method: '_validateStartRestore', - }); + log.debug('The object is currently being restored.', { + archive: objectMD.archive, + method: '_validateStartRestore', + }); return errors.RestoreAlreadyInProgress; } return undefined; @@ -142,21 +137,24 @@ function validatePutVersionId(objMD, versionId, log) { const isLocationCold = locationConstraints[objMD.dataStoreName]?.isCold; if (!isLocationCold) { - log.error('The object data is not stored in a cold storage location.', - { - isLocationCold, - dataStoreName: objMD.dataStoreName, - method: 'validatePutVersionId', - }); + log.error('The object data is not stored in a cold storage location.', { + isLocationCold, + dataStoreName: objMD.dataStoreName, + method: 'validatePutVersionId', + }); return errors.InvalidObjectState; } // make sure object archive restoration is in progress // NOTE: we do not use putObjectVersion to update the restoration period. - if (!objMD.archive || !objMD.archive.restoreRequestedAt || !objMD.archive.restoreRequestedDays - || objMD.archive.restoreCompletedAt || objMD.archive.restoreWillExpireAt) { - log.error('object archive restoration is not in progress', - { method: 'validatePutVersionId', versionId }); + if ( + !objMD.archive || + !objMD.archive.restoreRequestedAt || + !objMD.archive.restoreRequestedDays || + objMD.archive.restoreCompletedAt || + objMD.archive.restoreWillExpireAt + ) { + log.error('object archive restoration is not in progress', { method: 'validatePutVersionId', versionId }); return errors.InvalidObjectState; } @@ -180,11 +178,11 @@ function _updateObjectExpirationDate(objectMD, log) { const isObjectAlreadyRestored = !!objectMD.archive.restoreCompletedAt; log.debug('The restore status of the object.', { isObjectAlreadyRestored, - method: 'isObjectAlreadyRestored' + method: 'isObjectAlreadyRestored', }); if (isObjectAlreadyRestored) { const expiryDate = new Date(objectMD.archive.restoreRequestedAt); - expiryDate.setTime(expiryDate.getTime() + (objectMD.archive.restoreRequestedDays * scaledMsPerDay)); + expiryDate.setTime(expiryDate.getTime() + objectMD.archive.restoreRequestedDays * scaledMsPerDay); /* eslint-disable no-param-reassign */ objectMD.archive.restoreWillExpireAt = expiryDate; @@ -209,9 +207,9 @@ function _updateObjectExpirationDate(objectMD, log) { */ function _updateRestoreInfo(objectMD, restoreParam, log) { if (!objectMD.archive) { - log.debug('objectMD.archive doesn\'t exits', { + log.debug("objectMD.archive doesn't exits", { objectMD, - method: '_updateRestoreInfo' + method: '_updateRestoreInfo', }); return errorInstances.InternalError.customizeDescription('Archive metadata is missing.'); } @@ -223,7 +221,7 @@ function _updateRestoreInfo(objectMD, restoreParam, log) { if (!ObjectMDArchive.isValid(objectMD.archive)) { log.debug('archive is not valid', { archive: objectMD.archive, - method: '_updateRestoreInfo' + method: '_updateRestoreInfo', }); return errorInstances.InternalError.customizeDescription('Invalid archive metadata.'); } @@ -249,7 +247,7 @@ function startRestore(objectMD, restoreParam, log, cb) { if (checkResultError) { log.debug('Restore cannot be done.', { error: checkResultError, - method: 'startRestore' + method: 'startRestore', }); return cb(checkResultError); } @@ -257,12 +255,12 @@ function startRestore(objectMD, restoreParam, log, cb) { if (updateResultError) { log.debug('Failed to update restore information.', { error: updateResultError, - method: 'startRestore' + method: 'startRestore', }); return cb(updateResultError); } log.debug('Validated and updated restore information', { - method: 'startRestore' + method: 'startRestore', }); const isObjectAlreadyRestored = _updateObjectExpirationDate(objectMD, log); return cb(null, isObjectAlreadyRestored); @@ -275,13 +273,16 @@ function startRestore(objectMD, restoreParam, log, cb) { */ function verifyColdObjectAvailable(objMD) { // return error when object is cold - if (objMD.archive && + if ( + objMD.archive && // Object is in cold backend (!objMD.archive.restoreRequestedAt || // Object is being restored - (objMD.archive.restoreRequestedAt && !objMD.archive.restoreCompletedAt))) { - const err = errorInstances.InvalidObjectState - .customizeDescription('The operation is not valid for the object\'s storage class'); + (objMD.archive.restoreRequestedAt && !objMD.archive.restoreCompletedAt)) + ) { + const err = errorInstances.InvalidObjectState.customizeDescription( + "The operation is not valid for the object's storage class", + ); return err; } return null; diff --git a/lib/api/apiUtils/object/corsResponse.js b/lib/api/apiUtils/object/corsResponse.js index 01020b5122..e8cd86df39 100644 --- a/lib/api/apiUtils/object/corsResponse.js +++ b/lib/api/apiUtils/object/corsResponse.js @@ -1,9 +1,9 @@ /** _matchesValue - compare two values to determine if they match -* @param {string} allowedValue - an allowed value in a CORS rule; -* may contain wildcards -* @param {string} value - value from CORS request -* @return {boolean} - true/false -*/ + * @param {string} allowedValue - an allowed value in a CORS rule; + * may contain wildcards + * @param {string} value - value from CORS request + * @return {boolean} - true/false + */ function _matchesValue(allowedValue, value) { const wildcardIndex = allowedValue.indexOf('*'); // If no wildcards, simply return whether strings are equal @@ -14,27 +14,28 @@ function _matchesValue(allowedValue, value) { // and after the wildcard const beginValue = allowedValue.substring(0, wildcardIndex); const endValue = allowedValue.substring(wildcardIndex + 1); - return (value.startsWith(beginValue) && value.endsWith(endValue)); + return value.startsWith(beginValue) && value.endsWith(endValue); } /** _matchesOneOf - check if header matches any AllowedHeaders of a rule -* @param {string[]} allowedHeaders - headers allowed in CORS rule -* @param {string} header - header from CORS request -* @return {boolean} - true/false -*/ + * @param {string[]} allowedHeaders - headers allowed in CORS rule + * @param {string} header - header from CORS request + * @return {boolean} - true/false + */ function _matchesOneOf(allowedHeaders, header) { return allowedHeaders.some(allowedHeader => // AllowedHeaders may have been stored with uppercase letters // during putBucketCors; ignore case when searching for match - _matchesValue(allowedHeader.toLowerCase(), header)); + _matchesValue(allowedHeader.toLowerCase(), header), + ); } /** _headersMatchRule - check if headers match AllowedHeaders of rule -* @param {string[]} headers - the value of the 'Access-Control-Request-Headers' -* in an OPTIONS request -* @param {string[]} allowedHeaders - AllowedHeaders of a CORS rule -* @return {boolean} - true/false -*/ + * @param {string[]} headers - the value of the 'Access-Control-Request-Headers' + * in an OPTIONS request + * @param {string[]} allowedHeaders - AllowedHeaders of a CORS rule + * @return {boolean} - true/false + */ function _headersMatchRule(headers, allowedHeaders) { if (!allowedHeaders) { return false; @@ -46,33 +47,31 @@ function _headersMatchRule(headers, allowedHeaders) { } /** _findCorsRule - Return first matching rule in cors rules that permits -* CORS request -* @param {object[]} rules - array of rules -* @param {string} [rules.id] - optional id to identify rule -* @param {string[]} rules[].allowedMethods - methods allowed for CORS -* @param {string[]} rules[].allowedOrigins - origins allowed for CORS -* @param {string[]} [rules[].allowedHeaders] - headers allowed in an -* OPTIONS request via the Access-Control-Request-Headers header -* @param {number} [rules[].maxAgeSeconds] - seconds browsers should cache -* OPTIONS response -* @param {string[]} [rules[].exposeHeaders] - headers to expose to external -* applications -* @param {string} origin - origin of CORS request -* @param {string} method - Access-Control-Request-Method header value in -* an OPTIONS request and the actual method in any other request -* @param {string[]} [headers] - Access-Control-Request-Headers header value -* in a preflight CORS request -* @return {(null|object)} - matching rule if found; null if no match -*/ + * CORS request + * @param {object[]} rules - array of rules + * @param {string} [rules.id] - optional id to identify rule + * @param {string[]} rules[].allowedMethods - methods allowed for CORS + * @param {string[]} rules[].allowedOrigins - origins allowed for CORS + * @param {string[]} [rules[].allowedHeaders] - headers allowed in an + * OPTIONS request via the Access-Control-Request-Headers header + * @param {number} [rules[].maxAgeSeconds] - seconds browsers should cache + * OPTIONS response + * @param {string[]} [rules[].exposeHeaders] - headers to expose to external + * applications + * @param {string} origin - origin of CORS request + * @param {string} method - Access-Control-Request-Method header value in + * an OPTIONS request and the actual method in any other request + * @param {string[]} [headers] - Access-Control-Request-Headers header value + * in a preflight CORS request + * @return {(null|object)} - matching rule if found; null if no match + */ function findCorsRule(rules, origin, method, headers) { return rules.find(rule => { if (rule.allowedMethods.indexOf(method) === -1) { return false; - } else if (!rule.allowedOrigins.some(allowedOrigin => - _matchesValue(allowedOrigin, origin))) { + } else if (!rule.allowedOrigins.some(allowedOrigin => _matchesValue(allowedOrigin, origin))) { return false; - } else if (headers && - !_headersMatchRule(headers, rule.allowedHeaders)) { + } else if (headers && !_headersMatchRule(headers, rule.allowedHeaders)) { return false; } return true; @@ -80,32 +79,30 @@ function findCorsRule(rules, origin, method, headers) { } /** _gatherResHeaders - Collect headers to return in response -* @param {object} rule - array of rules -* @param {string} [rule.id] - optional id to identify rule -* @param {string[]} rule[].allowedMethods - methods allowed for CORS -* @param {string[]} rule[].allowedOrigins - origins allowed for CORS -* @param {string[]} [rule[].allowedHeaders] - headers allowed in an -* OPTIONS request via the Access-Control-Request-Headers header -* @param {number} [rule[].maxAgeSeconds] - seconds browsers should cache -* OPTIONS response -* @param {string[]} [rule[].exposeHeaders] - headers to expose to external -* applications -* @param {string} origin - origin of CORS request -* @param {string} method - Access-Control-Request-Method header value in -* an OPTIONS request and the actual method in any other request -* @param {string[]} [headers] - Access-Control-Request-Headers header value -* in a preflight CORS request -* @param {boolean} [isPreflight] - indicates if cors headers are being gathered -* for a CORS preflight request -* @return {object} resHeaders - headers to include in response -*/ -function generateCorsResHeaders(rule, origin, method, headers, -isPreflight) { + * @param {object} rule - array of rules + * @param {string} [rule.id] - optional id to identify rule + * @param {string[]} rule[].allowedMethods - methods allowed for CORS + * @param {string[]} rule[].allowedOrigins - origins allowed for CORS + * @param {string[]} [rule[].allowedHeaders] - headers allowed in an + * OPTIONS request via the Access-Control-Request-Headers header + * @param {number} [rule[].maxAgeSeconds] - seconds browsers should cache + * OPTIONS response + * @param {string[]} [rule[].exposeHeaders] - headers to expose to external + * applications + * @param {string} origin - origin of CORS request + * @param {string} method - Access-Control-Request-Method header value in + * an OPTIONS request and the actual method in any other request + * @param {string[]} [headers] - Access-Control-Request-Headers header value + * in a preflight CORS request + * @param {boolean} [isPreflight] - indicates if cors headers are being gathered + * for a CORS preflight request + * @return {object} resHeaders - headers to include in response + */ +function generateCorsResHeaders(rule, origin, method, headers, isPreflight) { const resHeaders = { 'access-control-max-age': rule.maxAgeSeconds, 'access-control-allow-methods': rule.allowedMethods.join(', '), - 'vary': - 'Origin, Access-Control-Request-Headers, Access-Control-Request-Method', + vary: 'Origin, Access-Control-Request-Headers, Access-Control-Request-Method', }; // send back '*' if any origin allowed; otherwise send back // request Origin value @@ -121,8 +118,7 @@ isPreflight) { resHeaders['access-control-allow-headers'] = headers.join(', '); } if (rule.exposeHeaders) { - resHeaders['access-control-expose-headers'] = - rule.exposeHeaders.join(', '); + resHeaders['access-control-expose-headers'] = rule.exposeHeaders.join(', '); } if (isPreflight) { resHeaders['content-length'] = 0; diff --git a/lib/api/apiUtils/object/createAndStoreObject.js b/lib/api/apiUtils/object/createAndStoreObject.js index 542b9f5296..7b4bf0ffee 100644 --- a/lib/api/apiUtils/object/createAndStoreObject.js +++ b/lib/api/apiUtils/object/createAndStoreObject.js @@ -11,30 +11,34 @@ const { versioningPreprocessing, overwritingVersioning, decodeVID } = require('. const removeAWSChunked = require('./removeAWSChunked'); const getReplicationInfo = require('./getReplicationInfo'); const { config } = require('../../../Config'); -const validateWebsiteHeader = require('./websiteServing') - .validateWebsiteHeader; +const validateWebsiteHeader = require('./websiteServing').validateWebsiteHeader; const applyZenkoUserMD = require('./applyZenkoUserMD'); const { externalBackends, versioningNotImplBackends } = constants; -const externalVersioningErrorMessage = 'We do not currently support putting ' + -'a versioned object to a location-constraint of type Azure or GCP.'; +const externalVersioningErrorMessage = + 'We do not currently support putting ' + 'a versioned object to a location-constraint of type Azure or GCP.'; -function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle, - metadataStoreParams, dataToDelete, log, requestMethod, callback) { - services.metadataStoreObject(bucketName, dataGetInfo, - cipherBundle, metadataStoreParams, (err, result) => { - if (err) { - return callback(err); - } - if (dataToDelete) { - const newDataStoreName = Array.isArray(dataGetInfo) ? - dataGetInfo[0].dataStoreName : null; - return data.batchDelete(dataToDelete, requestMethod, - newDataStoreName, log, err => callback(err, result)); - } - return callback(null, result); - }); +function _storeInMDandDeleteData( + bucketName, + dataGetInfo, + cipherBundle, + metadataStoreParams, + dataToDelete, + log, + requestMethod, + callback, +) { + services.metadataStoreObject(bucketName, dataGetInfo, cipherBundle, metadataStoreParams, (err, result) => { + if (err) { + return callback(err); + } + if (dataToDelete) { + const newDataStoreName = Array.isArray(dataGetInfo) ? dataGetInfo[0].dataStoreName : null; + return data.batchDelete(dataToDelete, requestMethod, newDataStoreName, log, err => callback(err, result)); + } + return callback(null, result); + }); } /** createAndStoreObject - store data, store metadata, and delete old data @@ -59,9 +63,22 @@ function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle, * result.contentMD5 - content md5 of new object or version * result.versionId - unencrypted versionId returned by metadata */ -function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo, - canonicalID, cipherBundle, request, isDeleteMarker, streamingV4Params, - overheadField, log, originOp, callback) { +function createAndStoreObject( + bucketName, + bucketMD, + objectKey, + objMD, + authInfo, + canonicalID, + cipherBundle, + request, + isDeleteMarker, + streamingV4Params, + overheadField, + log, + originOp, + callback, +) { const putVersionId = request.headers['x-scal-s3-version-id']; const isPutVersion = putVersionId || putVersionId === ''; @@ -70,12 +87,10 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo, // delete marker, for our purposes we consider this to be a 'PUT' // operation const requestMethod = 'PUT'; - const websiteRedirectHeader = - request.headers['x-amz-website-redirect-location']; + const websiteRedirectHeader = request.headers['x-amz-website-redirect-location']; if (!validateWebsiteHeader(websiteRedirectHeader)) { const err = errors.InvalidRedirectLocation; - log.debug('invalid x-amz-website-redirect-location' + - `value ${websiteRedirectHeader}`, { error: err }); + log.debug('invalid x-amz-website-redirect-location' + `value ${websiteRedirectHeader}`, { error: err }); return callback(err); } @@ -115,8 +130,7 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo, size, headers, isDeleteMarker, - replicationInfo: getReplicationInfo(config, - objectKey, bucketMD, false, size, null, null, authInfo), + replicationInfo: getReplicationInfo(config, objectKey, bucketMD, false, size, null, null, authInfo), overheadField, log, }; @@ -141,17 +155,13 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo, if (!isDeleteMarker) { metadataStoreParams.contentType = request.headers['content-type']; metadataStoreParams.cacheControl = request.headers['cache-control']; - metadataStoreParams.contentDisposition = - request.headers['content-disposition']; - metadataStoreParams.contentEncoding = - removeAWSChunked(request.headers['content-encoding']); + metadataStoreParams.contentDisposition = request.headers['content-disposition']; + metadataStoreParams.contentEncoding = removeAWSChunked(request.headers['content-encoding']); metadataStoreParams.expires = request.headers.expires; metadataStoreParams.tagging = request.headers['x-amz-tagging']; - const defaultObjectLockConfiguration - = bucketMD.getObjectLockConfiguration(); + const defaultObjectLockConfiguration = bucketMD.getObjectLockConfiguration(); if (defaultObjectLockConfiguration) { - metadataStoreParams.defaultRetention - = defaultObjectLockConfiguration; + metadataStoreParams.defaultRetention = defaultObjectLockConfiguration; } } @@ -159,12 +169,10 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo, // the object's location constraint metaheader to determine backend info if (isDeleteMarker && objMD) { // eslint-disable-next-line no-param-reassign - request.headers[constants.objectLocationConstraintHeader] = - objMD[constants.objectLocationConstraintHeader]; + request.headers[constants.objectLocationConstraintHeader] = objMD[constants.objectLocationConstraintHeader]; } - const backendInfoObj = - locationConstraintCheck(request, null, bucketMD, log); + const backendInfoObj = locationConstraintCheck(request, null, bucketMD, log); if (backendInfoObj.err) { return process.nextTick(() => { callback(backendInfoObj.err); @@ -173,8 +181,7 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo, const backendInfo = backendInfoObj.backendInfo; const location = backendInfo.getControllingLocationConstraint(); - const locationType = backendInfoObj.defaultedToDataBackend ? location : - config.getLocationConstraintType(location); + const locationType = backendInfoObj.defaultedToDataBackend ? location : config.getLocationConstraintType(location); metadataStoreParams.dataStoreName = location; if (versioningNotImplBackends[locationType]) { @@ -182,11 +189,9 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo, const isVersionedObj = vcfg && vcfg.Status === 'Enabled'; if (isVersionedObj) { - log.debug(externalVersioningErrorMessage, - { method: 'createAndStoreObject', error: errors.NotImplemented }); + log.debug(externalVersioningErrorMessage, { method: 'createAndStoreObject', error: errors.NotImplemented }); return process.nextTick(() => { - callback(errorInstances.NotImplemented.customizeDescription( - externalVersioningErrorMessage)); + callback(errorInstances.NotImplemented.customizeDescription(externalVersioningErrorMessage)); }); } } @@ -212,125 +217,152 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo, const mdOnlyHeader = request.headers['x-amz-meta-mdonly']; const mdOnlySize = request.headers['x-amz-meta-size']; - return async.waterfall([ - function storeData(next) { - if (size === 0) { - if (!dontSkipBackend[locationType]) { - metadataStoreParams.contentMD5 = constants.emptyFileMd5; - return next(null, null, null); - } + return async.waterfall( + [ + function storeData(next) { + if (size === 0) { + if (!dontSkipBackend[locationType]) { + metadataStoreParams.contentMD5 = constants.emptyFileMd5; + return next(null, null, null); + } - // Handle mdOnlyHeader as a metadata only operation. If - // the object in question is actually 0 byte or has a body size - // then handle normally. - if (mdOnlyHeader === 'true' && mdOnlySize > 0) { - log.debug('metadata only operation x-amz-meta-mdonly'); - const md5 = request.headers['x-amz-meta-md5chksum'] - ? Buffer.from(request.headers['x-amz-meta-md5chksum'], - 'base64').toString('hex') : null; - const numParts = request.headers['x-amz-meta-md5numparts']; - let _md5; - if (numParts === undefined) { - _md5 = md5; - } else { - _md5 = `${md5}-${numParts}`; + // Handle mdOnlyHeader as a metadata only operation. If + // the object in question is actually 0 byte or has a body size + // then handle normally. + if (mdOnlyHeader === 'true' && mdOnlySize > 0) { + log.debug('metadata only operation x-amz-meta-mdonly'); + const md5 = request.headers['x-amz-meta-md5chksum'] + ? Buffer.from(request.headers['x-amz-meta-md5chksum'], 'base64').toString('hex') + : null; + const numParts = request.headers['x-amz-meta-md5numparts']; + let _md5; + if (numParts === undefined) { + _md5 = md5; + } else { + _md5 = `${md5}-${numParts}`; + } + const versionId = request.headers['x-amz-meta-version-id']; + const dataGetInfo = { + key: objectKey, + dataStoreName: location, + dataStoreType: locationType, + dataStoreVersionId: versionId, + dataStoreMD5: _md5, + }; + return next(null, dataGetInfo, _md5); } - const versionId = request.headers['x-amz-meta-version-id']; - const dataGetInfo = { - key: objectKey, - dataStoreName: location, - dataStoreType: locationType, - dataStoreVersionId: versionId, - dataStoreMD5: _md5, - }; - return next(null, dataGetInfo, _md5); } - } - return dataStore(objectKeyContext, cipherBundle, request, size, - streamingV4Params, backendInfo, log, next); - }, - function processDataResult(dataGetInfo, calculatedHash, next) { - if (dataGetInfo === null || dataGetInfo === undefined) { - return next(null, null); - } - // So that data retrieval information for MPU's and - // regular puts are stored in the same data structure, - // place the retrieval info here into a single element array - const { key, dataStoreName, dataStoreType, dataStoreETag, - dataStoreVersionId } = dataGetInfo; - const prefixedDataStoreETag = dataStoreETag - ? `1:${dataStoreETag}` - : `1:${calculatedHash}`; - const dataGetInfoArr = [{ key, size, start: 0, dataStoreName, - dataStoreType, dataStoreETag: prefixedDataStoreETag, - dataStoreVersionId }]; - if (cipherBundle) { - dataGetInfoArr[0].cryptoScheme = cipherBundle.cryptoScheme; - dataGetInfoArr[0].cipheredDataKey = - cipherBundle.cipheredDataKey; - } - if (mdOnlyHeader === 'true') { - metadataStoreParams.size = mdOnlySize; - dataGetInfoArr[0].size = mdOnlySize; - } - metadataStoreParams.contentMD5 = calculatedHash; - return next(null, dataGetInfoArr); - }, - function getVersioningInfo(infoArr, next) { - // if x-scal-s3-version-id header is specified, we overwrite the object/version metadata. - if (isPutVersion) { - const options = overwritingVersioning(objMD, metadataStoreParams); - return process.nextTick(() => next(null, options, infoArr)); - } + return dataStore( + objectKeyContext, + cipherBundle, + request, + size, + streamingV4Params, + backendInfo, + log, + next, + ); + }, + function processDataResult(dataGetInfo, calculatedHash, next) { + if (dataGetInfo === null || dataGetInfo === undefined) { + return next(null, null); + } + // So that data retrieval information for MPU's and + // regular puts are stored in the same data structure, + // place the retrieval info here into a single element array + const { key, dataStoreName, dataStoreType, dataStoreETag, dataStoreVersionId } = dataGetInfo; + const prefixedDataStoreETag = dataStoreETag ? `1:${dataStoreETag}` : `1:${calculatedHash}`; + const dataGetInfoArr = [ + { + key, + size, + start: 0, + dataStoreName, + dataStoreType, + dataStoreETag: prefixedDataStoreETag, + dataStoreVersionId, + }, + ]; + if (cipherBundle) { + dataGetInfoArr[0].cryptoScheme = cipherBundle.cryptoScheme; + dataGetInfoArr[0].cipheredDataKey = cipherBundle.cipheredDataKey; + } + if (mdOnlyHeader === 'true') { + metadataStoreParams.size = mdOnlySize; + dataGetInfoArr[0].size = mdOnlySize; + } + metadataStoreParams.contentMD5 = calculatedHash; + return next(null, dataGetInfoArr); + }, + function getVersioningInfo(infoArr, next) { + // if x-scal-s3-version-id header is specified, we overwrite the object/version metadata. + if (isPutVersion) { + const options = overwritingVersioning(objMD, metadataStoreParams); + return process.nextTick(() => next(null, options, infoArr)); + } - if (!bucketMD.isVersioningEnabled() && objMD?.archive?.archiveInfo) { - // Ensure we trigger a "delete" event in the oplog for the previously archived object - metadataStoreParams.needOplogUpdate = 's3:ReplaceArchivedObject'; - } + if (!bucketMD.isVersioningEnabled() && objMD?.archive?.archiveInfo) { + // Ensure we trigger a "delete" event in the oplog for the previously archived object + metadataStoreParams.needOplogUpdate = 's3:ReplaceArchivedObject'; + } - return versioningPreprocessing(bucketName, bucketMD, - metadataStoreParams.objectKey, objMD, log, (err, options) => { - if (err) { - // TODO: check AWS error when user requested a specific - // version before any versions have been put - const logLvl = err.is.BadRequest ? - 'debug' : 'error'; - log[logLvl]('error getting versioning info', { - error: err, - method: 'versioningPreprocessing', - }); - } + return versioningPreprocessing( + bucketName, + bucketMD, + metadataStoreParams.objectKey, + objMD, + log, + (err, options) => { + if (err) { + // TODO: check AWS error when user requested a specific + // version before any versions have been put + const logLvl = err.is.BadRequest ? 'debug' : 'error'; + log[logLvl]('error getting versioning info', { + error: err, + method: 'versioningPreprocessing', + }); + } - const location = infoArr?.[0]?.dataStoreName; - if (location === bucketMD.getLocationConstraint() && bucketMD.isIngestionBucket()) { - // If the object is being written to the "ingested" storage location, keep the same - // versionId for consistency and to avoid creating an extra version when it gets - // ingested - const backendVersionId = decodeVID(infoArr[0].dataStoreVersionId); - if (!(backendVersionId instanceof Error)) { - options.versionId = backendVersionId; // eslint-disable-line no-param-reassign + const location = infoArr?.[0]?.dataStoreName; + if (location === bucketMD.getLocationConstraint() && bucketMD.isIngestionBucket()) { + // If the object is being written to the "ingested" storage location, keep the same + // versionId for consistency and to avoid creating an extra version when it gets + // ingested + const backendVersionId = decodeVID(infoArr[0].dataStoreVersionId); + if (!(backendVersionId instanceof Error)) { + options.versionId = backendVersionId; // eslint-disable-line no-param-reassign + } } - } - return next(err, options, infoArr); - }); - }, - function storeMDAndDeleteData(options, infoArr, next) { - metadataStoreParams.versionId = options.versionId; - metadataStoreParams.versioning = options.versioning; - metadataStoreParams.isNull = options.isNull; - metadataStoreParams.deleteNullKey = options.deleteNullKey; + return next(err, options, infoArr); + }, + ); + }, + function storeMDAndDeleteData(options, infoArr, next) { + metadataStoreParams.versionId = options.versionId; + metadataStoreParams.versioning = options.versioning; + metadataStoreParams.isNull = options.isNull; + metadataStoreParams.deleteNullKey = options.deleteNullKey; - if (options.extraMD) { - Object.assign(metadataStoreParams, options.extraMD); - } + if (options.extraMD) { + Object.assign(metadataStoreParams, options.extraMD); + } - return _storeInMDandDeleteData(bucketName, infoArr, - cipherBundle, metadataStoreParams, - options.dataToDelete, log, requestMethod, next); - }, - ], callback); + return _storeInMDandDeleteData( + bucketName, + infoArr, + cipherBundle, + metadataStoreParams, + options.dataToDelete, + log, + requestMethod, + next, + ); + }, + ], + callback, + ); } module.exports = createAndStoreObject; diff --git a/lib/api/apiUtils/object/expirationHeaders.js b/lib/api/apiUtils/object/expirationHeaders.js index 6edcedeb9d..36361f4f95 100644 --- a/lib/api/apiUtils/object/expirationHeaders.js +++ b/lib/api/apiUtils/object/expirationHeaders.js @@ -1,16 +1,8 @@ const { LifecycleConfiguration } = require('arsenal').models; -const { - LifecycleDateTime, - LifecycleUtils, -} = require('arsenal').s3middleware.lifecycleHelpers; +const { LifecycleDateTime, LifecycleUtils } = require('arsenal').s3middleware.lifecycleHelpers; const { config } = require('../../../Config'); -const { - expireOneDayEarlier, - transitionOneDayEarlier, - timeProgressionFactor, - scaledMsPerDay, -} = config.getTimeOptions(); +const { expireOneDayEarlier, transitionOneDayEarlier, timeProgressionFactor, scaledMsPerDay } = config.getTimeOptions(); const lifecycleDateTime = new LifecycleDateTime({ transitionOneDayEarlier, @@ -21,7 +13,7 @@ const lifecycleDateTime = new LifecycleDateTime({ const lifecycleUtils = new LifecycleUtils(config.supportedLifecycleRules, lifecycleDateTime, timeProgressionFactor); function calculateDate(objDate, expDays, datetime) { - return new Date(datetime.getTimestamp(objDate) + (expDays * scaledMsPerDay)); + return new Date(datetime.getTimestamp(objDate) + expDays * scaledMsPerDay); } function formatExpirationHeader(date, id) { @@ -35,13 +27,9 @@ const AMZ_ABORT_DATE_HEADER = 'x-amz-abort-date'; // format: x-amz-abort-rule-id: "rule id" const AMZ_ABORT_ID_HEADER = 'x-amz-abort-rule-id'; - function _generateExpHeadersObjects(rules, params, datetime) { const tags = { - TagSet: params.tags - ? Object.keys(params.tags) - .map(key => ({ Key: key, Value: params.tags[key] })) - : [], + TagSet: params.tags ? Object.keys(params.tags).map(key => ({ Key: key, Value: params.tags[key] })) : [], }; const objectInfo = { Key: params.key }; @@ -80,11 +68,7 @@ function _generateExpHeadresMPU(rules, params, datetime) { if (applicable.AbortIncompleteMultipartUpload) { const rule = applicable.AbortIncompleteMultipartUpload; - const date = calculateDate( - params.date, - rule.DaysAfterInitiation, - datetime - ); + const date = calculateDate(params.date, rule.DaysAfterInitiation, datetime); return { [AMZ_ABORT_ID_HEADER]: encodeURIComponent(rule.ID), diff --git a/lib/api/apiUtils/object/getReplicationBackendDataLocator.js b/lib/api/apiUtils/object/getReplicationBackendDataLocator.js index b5ba4956c8..f04d011ba1 100644 --- a/lib/api/apiUtils/object/getReplicationBackendDataLocator.js +++ b/lib/api/apiUtils/object/getReplicationBackendDataLocator.js @@ -26,25 +26,26 @@ const { errorInstances } = require('arsenal'); */ function getReplicationBackendDataLocator(locationObj, replicationInfo) { const repBackendResult = {}; - const locMatch = replicationInfo.backends.find( - backend => backend.site === locationObj.location); + const locMatch = replicationInfo.backends.find(backend => backend.site === locationObj.location); if (!locMatch) { - repBackendResult.error = errorInstances.InvalidLocationConstraint. - customizeDescription('Object is not replicated to location ' + - 'passed in location header'); + repBackendResult.error = errorInstances.InvalidLocationConstraint.customizeDescription( + 'Object is not replicated to location ' + 'passed in location header', + ); return repBackendResult; } repBackendResult.status = locMatch.status; if (['PENDING', 'FAILED'].includes(locMatch.status)) { - repBackendResult.reason = - `Object replication to specified backend is ${locMatch.status}`; + repBackendResult.reason = `Object replication to specified backend is ${locMatch.status}`; return repBackendResult; } - repBackendResult.dataLocator = [{ - key: locationObj.key, - dataStoreName: locationObj.location, - dataStoreType: locationObj.locationType, - dataStoreVersionId: locMatch.dataStoreVersionId }]; + repBackendResult.dataLocator = [ + { + key: locationObj.key, + dataStoreName: locationObj.location, + dataStoreType: locationObj.locationType, + dataStoreVersionId: locMatch.dataStoreVersionId, + }, + ]; return repBackendResult; } diff --git a/lib/api/apiUtils/object/getReplicationInfo.js b/lib/api/apiUtils/object/getReplicationInfo.js index b7e4afb25d..f9f8b92403 100644 --- a/lib/api/apiUtils/object/getReplicationInfo.js +++ b/lib/api/apiUtils/object/getReplicationInfo.js @@ -1,5 +1,4 @@ -const { isServiceAccount, getServiceAccountProperties } = - require('../authorization/permissionChecks'); +const { isServiceAccount, getServiceAccountProperties } = require('../authorization/permissionChecks'); const { replicationBackends } = require('arsenal').constants; function _getBackend(objectMD, site) { @@ -23,15 +22,13 @@ function _getStorageClasses(s3config, rule) { const { replicationEndpoints } = s3config; // If no storage class, use the given default endpoint or the sole endpoint if (replicationEndpoints.length > 0) { - const endPoint = - replicationEndpoints.find(endpoint => endpoint.default) || replicationEndpoints[0]; + const endPoint = replicationEndpoints.find(endpoint => endpoint.default) || replicationEndpoints[0]; return [endPoint.site]; } return undefined; } -function _getReplicationInfo(s3config, rule, replicationConfig, content, operationType, - objectMD, bucketMD) { +function _getReplicationInfo(s3config, rule, replicationConfig, content, operationType, objectMD, bucketMD) { const storageTypes = []; const backends = []; const storageClasses = _getStorageClasses(s3config, rule); @@ -39,9 +36,7 @@ function _getReplicationInfo(s3config, rule, replicationConfig, content, operati return undefined; } storageClasses.forEach(storageClass => { - const storageClassName = - storageClass.endsWith(':preferred_read') ? - storageClass.split(':')[0] : storageClass; + const storageClassName = storageClass.endsWith(':preferred_read') ? storageClass.split(':')[0] : storageClass; // TODO CLDSRV-646: for consistency, should we look at replicationEndpoints instead, like // `_getStorageClasses()` ? const location = s3config.locationConstraints[storageClassName]; @@ -80,8 +75,7 @@ function _getReplicationInfo(s3config, rule, replicationConfig, content, operati * @param {AuthInfo} [authInfo] - authentication info of object owner * @return {undefined} */ -function getReplicationInfo( - s3config, objKey, bucketMD, isMD, objSize, operationType, objectMD, authInfo) { +function getReplicationInfo(s3config, objKey, bucketMD, isMD, objSize, operationType, objectMD, authInfo) { const content = isMD || objSize === 0 ? ['METADATA'] : ['DATA', 'METADATA']; const config = bucketMD.getReplicationConfiguration(); @@ -106,17 +100,14 @@ function getReplicationInfo( if (!authInfo || !isServiceAccount(authInfo.getCanonicalID())) { doReplicate = true; } else { - const serviceAccountProps = getServiceAccountProperties( - authInfo.getCanonicalID()); + const serviceAccountProps = getServiceAccountProperties(authInfo.getCanonicalID()); doReplicate = serviceAccountProps.canReplicate; } if (doReplicate) { - const rule = config.rules.find( - rule => (objKey.startsWith(rule.prefix) && rule.enabled)); + const rule = config.rules.find(rule => objKey.startsWith(rule.prefix) && rule.enabled); if (rule) { // TODO CLDSRV-646 : should "merge" the replicationInfo for different rules - return _getReplicationInfo( - s3config, rule, config, content, operationType, objectMD, bucketMD); + return _getReplicationInfo(s3config, rule, config, content, operationType, objectMD, bucketMD); } } } diff --git a/lib/api/apiUtils/object/locationConstraintCheck.js b/lib/api/apiUtils/object/locationConstraintCheck.js index bc87fc249b..cb12ce3531 100644 --- a/lib/api/apiUtils/object/locationConstraintCheck.js +++ b/lib/api/apiUtils/object/locationConstraintCheck.js @@ -20,28 +20,33 @@ function locationConstraintCheck(request, metaHeaders, bucket, log) { let objectLocationConstraint; if (metaHeaders) { - objectLocationConstraint = - metaHeaders[constants.objectLocationConstraintHeader]; + objectLocationConstraint = metaHeaders[constants.objectLocationConstraintHeader]; } else { - objectLocationConstraint = request - .headers[constants.objectLocationConstraintHeader]; + objectLocationConstraint = request.headers[constants.objectLocationConstraintHeader]; } const bucketLocationConstraint = bucket.getLocationConstraint(); const requestEndpoint = request.parsedHost; - const controllingBackend = BackendInfo.controllingBackendParam(config, - objectLocationConstraint, bucketLocationConstraint, - requestEndpoint, log); + const controllingBackend = BackendInfo.controllingBackendParam( + config, + objectLocationConstraint, + bucketLocationConstraint, + requestEndpoint, + log, + ); if (!controllingBackend.isValid) { backendInfoObj = { - err: errorInstances.InvalidArgument.customizeDescription(controllingBackend. - description), + err: errorInstances.InvalidArgument.customizeDescription(controllingBackend.description), }; return backendInfoObj; } - const backendInfo = new BackendInfo(config, objectLocationConstraint, - bucketLocationConstraint, requestEndpoint, - controllingBackend.legacyLocationConstraint); + const backendInfo = new BackendInfo( + config, + objectLocationConstraint, + bucketLocationConstraint, + requestEndpoint, + controllingBackend.legacyLocationConstraint, + ); backendInfoObj = { err: null, controllingLC: backendInfo.getControllingLocationConstraint(), diff --git a/lib/api/apiUtils/object/locationHeaderCheck.js b/lib/api/apiUtils/object/locationHeaderCheck.js index 6fc3ec24f1..d29baa5b57 100644 --- a/lib/api/apiUtils/object/locationHeaderCheck.js +++ b/lib/api/apiUtils/object/locationHeaderCheck.js @@ -19,11 +19,11 @@ function locationHeaderCheck(headers, objectKey, bucketName) { const validLocation = config.locationConstraints[location]; if (!validLocation) { return errorInstances.InvalidLocationConstraint.customizeDescription( - 'Invalid location constraint specified in header'); + 'Invalid location constraint specified in header', + ); } const bucketMatch = validLocation.details.bucketMatch; - const backendKey = bucketMatch ? objectKey : - `${bucketName}/${objectKey}`; + const backendKey = bucketMatch ? objectKey : `${bucketName}/${objectKey}`; return { location, key: backendKey, diff --git a/lib/api/apiUtils/object/locationKeysHaveChanged.js b/lib/api/apiUtils/object/locationKeysHaveChanged.js index 41c8560023..0f372cefdd 100644 --- a/lib/api/apiUtils/object/locationKeysHaveChanged.js +++ b/lib/api/apiUtils/object/locationKeysHaveChanged.js @@ -1,18 +1,18 @@ /** -* Check if all keys that exist in the current list which will be used -* in composing object are not present in the old object's list. -* -* This method can be used to check against accidentally removing data -* keys due to instability from the metadata layer, or for replay -* detection in general. -* -* @param {array|string|null} prev - list of keys from the object being -* overwritten -* @param {array|null} curr - list of keys to be used in composing -* current object -* @returns {boolean} true if no key in `curr` is present in `prev`, -* false otherwise -*/ + * Check if all keys that exist in the current list which will be used + * in composing object are not present in the old object's list. + * + * This method can be used to check against accidentally removing data + * keys due to instability from the metadata layer, or for replay + * detection in general. + * + * @param {array|string|null} prev - list of keys from the object being + * overwritten + * @param {array|null} curr - list of keys to be used in composing + * current object + * @returns {boolean} true if no key in `curr` is present in `prev`, + * false otherwise + */ function locationKeysHaveChanged(prev, curr) { if (!prev || prev.length === 0 || !curr) { return true; diff --git a/lib/api/apiUtils/object/locationStorageCheck.js b/lib/api/apiUtils/object/locationStorageCheck.js index 88cbbb18a3..6129902377 100644 --- a/lib/api/apiUtils/object/locationStorageCheck.js +++ b/lib/api/apiUtils/object/locationStorageCheck.js @@ -1,8 +1,7 @@ const { errorInstances } = require('arsenal'); const { config } = require('../../../Config'); -const { getLocationMetric, pushLocationMetric } = - require('../../../utapi/utilities'); +const { getLocationMetric, pushLocationMetric } = require('../../../utapi/utilities'); function _gbToBytes(gb) { return gb * 1024 * 1024 * 1024; @@ -37,9 +36,11 @@ function locationStorageCheck(location, updateSize, log, cb) { const newStorageSize = parseInt(bytesStored, 10) + updateSize; const sizeLimitBytes = _gbToBytes(sizeLimitGB); if (sizeLimitBytes < newStorageSize) { - return cb(errorInstances.AccessDenied.customizeDescription( - `The assigned storage space limit for location ${location} ` + - 'will be exceeded')); + return cb( + errorInstances.AccessDenied.customizeDescription( + `The assigned storage space limit for location ${location} ` + 'will be exceeded', + ), + ); } return pushLocationMetric(location, updateSize, log, cb); }); diff --git a/lib/api/apiUtils/object/objectAttributes.js b/lib/api/apiUtils/object/objectAttributes.js index 1f7ddcf44a..8fa93a781e 100644 --- a/lib/api/apiUtils/object/objectAttributes.js +++ b/lib/api/apiUtils/object/objectAttributes.js @@ -58,54 +58,50 @@ function parseAttributesHeaders(headers, headerName, supportedAttributes) { * @returns {void} - this function does not return a value, it mutates the `xml` param. */ function buildAttributesXml(objectMD, userMetadata, requestedAttrs, xml) { - const customAttributes = new Set(); - for (const attribute of requestedAttrs) { - switch (attribute) { - case 'ETag': - xml.push(`${objectMD['content-md5']}`); - break; - case 'ObjectParts': { - const partCount = getPartCountFromMd5(objectMD); - if (partCount) { - xml.push( - '', - `${partCount}`, - '', - ); - } - break; - } - case 'StorageClass': - xml.push(`${objectMD['x-amz-storage-class']}`); - break; - case 'ObjectSize': - xml.push(`${objectMD['content-length']}`); - break; - case 'RestoreStatus': - xml.push(''); - xml.push(`${!!objectMD.restoreStatus?.inProgress}`); + const customAttributes = new Set(); + for (const attribute of requestedAttrs) { + switch (attribute) { + case 'ETag': + xml.push(`${objectMD['content-md5']}`); + break; + case 'ObjectParts': { + const partCount = getPartCountFromMd5(objectMD); + if (partCount) { + xml.push('', `${partCount}`, ''); + } + break; + } + case 'StorageClass': + xml.push(`${objectMD['x-amz-storage-class']}`); + break; + case 'ObjectSize': + xml.push(`${objectMD['content-length']}`); + break; + case 'RestoreStatus': + xml.push(''); + xml.push(`${!!objectMD.restoreStatus?.inProgress}`); - if (objectMD.restoreStatus?.expiryDate) { - xml.push(`${objectMD.restoreStatus?.expiryDate}`); - } + if (objectMD.restoreStatus?.expiryDate) { + xml.push(`${objectMD.restoreStatus?.expiryDate}`); + } - xml.push(''); - break; - case 'x-amz-meta-*': - for (const key of Object.keys(userMetadata)) { - customAttributes.add(key); - } - break; - default: - if (userMetadata[attribute]) { - customAttributes.add(attribute); + xml.push(''); + break; + case 'x-amz-meta-*': + for (const key of Object.keys(userMetadata)) { + customAttributes.add(key); + } + break; + default: + if (userMetadata[attribute]) { + customAttributes.add(attribute); + } } } - } - for (const key of customAttributes) { - xml.push(`<${key}>${userMetadata[key]}`); - } + for (const key of customAttributes) { + xml.push(`<${key}>${userMetadata[key]}`); + } } module.exports = { diff --git a/lib/api/apiUtils/object/objectLockHelpers.js b/lib/api/apiUtils/object/objectLockHelpers.js index e2e18fc640..ede44835dc 100644 --- a/lib/api/apiUtils/object/objectLockHelpers.js +++ b/lib/api/apiUtils/object/objectLockHelpers.js @@ -22,8 +22,7 @@ function calculateRetainUntilDate(retention) { // Calculate the number of days to retain the lock on the object const retainUntilDays = days || years * 365; const retainUntilDaysInMs = retainUntilDays * scaledMsPerDay; - const retainUntilDate - = date.add(retainUntilDaysInMs, 'ms'); + const retainUntilDate = date.add(retainUntilDaysInMs, 'ms'); return retainUntilDate.toISOString(); } /** @@ -40,33 +39,26 @@ function validateHeaders(bucket, headers, log) { const objectLockMode = headers['x-amz-object-lock-mode']; // If retention headers or legal hold header present but // object lock is not enabled on the bucket return error - if ((objectLockDate || objectLockMode || objectLegalHold) - && !bucketObjectLockEnabled) { + if ((objectLockDate || objectLockMode || objectLegalHold) && !bucketObjectLockEnabled) { log.trace('bucket is missing ObjectLockConfiguration'); - return errorInstances.InvalidRequest.customizeDescription( - 'Bucket is missing ObjectLockConfiguration'); + return errorInstances.InvalidRequest.customizeDescription('Bucket is missing ObjectLockConfiguration'); } - if ((objectLockMode || objectLockDate) && - !(objectLockMode && objectLockDate)) { + if ((objectLockMode || objectLockDate) && !(objectLockMode && objectLockDate)) { return errorInstances.InvalidArgument.customizeDescription( - 'x-amz-object-lock-retain-until-date and ' + - 'x-amz-object-lock-mode must both be supplied', + 'x-amz-object-lock-retain-until-date and ' + 'x-amz-object-lock-mode must both be supplied', ); } const validModes = new Set(['GOVERNANCE', 'COMPLIANCE']); if (objectLockMode && !validModes.has(objectLockMode)) { - return errorInstances.InvalidArgument.customizeDescription( - 'Unknown wormMode directive'); + return errorInstances.InvalidArgument.customizeDescription('Unknown wormMode directive'); } const validLegalHolds = new Set(['ON', 'OFF']); if (objectLegalHold && !validLegalHolds.has(objectLegalHold)) { - return errorInstances.InvalidArgument.customizeDescription( - 'Legal hold status must be one of "ON", "OFF"'); + return errorInstances.InvalidArgument.customizeDescription('Legal hold status must be one of "ON", "OFF"'); } const currentDate = new Date().toISOString(); if (objectLockMode && objectLockDate <= currentDate) { - return errorInstances.InvalidArgument.customizeDescription( - 'The retain until date must be in the future!'); + return errorInstances.InvalidArgument.customizeDescription('The retain until date must be in the future!'); } return null; } @@ -121,8 +113,7 @@ function compareObjectLockInformation(headers, defaultRetention) { function setObjectLockInformation(headers, md, defaultRetention) { // Stores retention information if object either has its own retention // configuration or default retention configuration from its bucket - const finalObjectLockInfo = - compareObjectLockInformation(headers, defaultRetention); + const finalObjectLockInfo = compareObjectLockInformation(headers, defaultRetention); if (finalObjectLockInfo.retentionInfo) { md.setRetentionMode(finalObjectLockInfo.retentionInfo.mode); md.setRetentionDate(finalObjectLockInfo.retentionInfo.date); @@ -261,7 +252,6 @@ function hasGovernanceBypassHeader(headers) { return bypassHeader.toLowerCase() === 'true'; } - /** * checkUserGovernanceBypass * @@ -276,10 +266,9 @@ function hasGovernanceBypassHeader(headers) { * @returns {undefined} - */ function checkUserGovernanceBypass(request, authInfo, bucketMD, objectKey, log, cb) { - log.trace( - 'object in GOVERNANCE mode and is user, checking for attached policies', - { method: 'checkUserPolicyGovernanceBypass' }, - ); + log.trace('object in GOVERNANCE mode and is user, checking for attached policies', { + method: 'checkUserPolicyGovernanceBypass', + }); const authParams = auth.server.extractParams(request, log, 's3', request.query); const ip = policies.requestUtils.getClientIp(request, config); @@ -301,41 +290,41 @@ function checkUserGovernanceBypass(request, authInfo, bucketMD, objectKey, log, signatureAge: authParams.params.data.signatureAge, }, }; - return vault.checkPolicies(requestContextParams, - authInfo.getArn(), log, (err, authorizationResults) => { - if (err) { - return cb(err); - } - const explicitDenyExists = authorizationResults.some( - authzResult => authzResult.isAllowed === false && !authzResult.isImplicit); - if (explicitDenyExists) { - log.trace('authorization check failed for user', - { - 'method': 'checkUserPolicyGovernanceBypass', - 's3:BypassGovernanceRetention': false, - }); - return cb(errors.AccessDenied); - } - // Convert authorization results into an easier to handle format - const actionImplicitDenies = authorizationResults.reduce((acc, curr, idx) => { - const apiMethod = authorizationResults[idx].action; - // eslint-disable-next-line no-param-reassign - acc[apiMethod] = curr.isImplicit; - return acc; - }, {}); + return vault.checkPolicies(requestContextParams, authInfo.getArn(), log, (err, authorizationResults) => { + if (err) { + return cb(err); + } + const explicitDenyExists = authorizationResults.some( + authzResult => authzResult.isAllowed === false && !authzResult.isImplicit, + ); + if (explicitDenyExists) { + log.trace('authorization check failed for user', { + method: 'checkUserPolicyGovernanceBypass', + 's3:BypassGovernanceRetention': false, + }); + return cb(errors.AccessDenied); + } + // Convert authorization results into an easier to handle format + const actionImplicitDenies = authorizationResults.reduce((acc, curr, idx) => { + const apiMethod = authorizationResults[idx].action; + // eslint-disable-next-line no-param-reassign + acc[apiMethod] = curr.isImplicit; + return acc; + }, {}); - // Evaluate against the bucket policies - const areAllActionsAllowed = evaluateBucketPolicyWithIAM( - bucketMD, - Object.keys(actionImplicitDenies), - authInfo.getCanonicalID(), - authInfo, - actionImplicitDenies, - log, - request); + // Evaluate against the bucket policies + const areAllActionsAllowed = evaluateBucketPolicyWithIAM( + bucketMD, + Object.keys(actionImplicitDenies), + authInfo.getCanonicalID(), + authInfo, + actionImplicitDenies, + log, + request, + ); - return cb(areAllActionsAllowed === true ? null : errors.AccessDenied); - }); + return cb(areAllActionsAllowed === true ? null : errors.AccessDenied); + }); } module.exports = { diff --git a/lib/api/apiUtils/object/objectRestore.js b/lib/api/apiUtils/object/objectRestore.js index 7912416258..bfa3ce0301 100644 --- a/lib/api/apiUtils/object/objectRestore.js +++ b/lib/api/apiUtils/object/objectRestore.js @@ -42,12 +42,11 @@ function objectRestore(metadata, mdUtils, userInfo, request, log, callback) { const decodedVidResult = decodeVersionId(request.query); if (decodedVidResult instanceof Error) { - log.trace('invalid versionId query', - { - method: METHOD, - versionId: request.query.versionId, - error: decodedVidResult, - }); + log.trace('invalid versionId query', { + method: METHOD, + versionId: request.query.versionId, + error: decodedVidResult, + }); return process.nextTick(() => callback(decodedVidResult)); } @@ -69,36 +68,41 @@ function objectRestore(metadata, mdUtils, userInfo, request, log, callback) { request, }; - return async.waterfall([ + return async.waterfall( + [ // get metadata of bucket and object function validateBucketAndObject(next) { - return mdUtils.standardMetadataValidateBucketAndObj(mdValueParams, request.actionImplicitDenies, - log, (err, bucketMD, objectMD) => { - if (err) { - log.trace('request authorization failed', { method: METHOD, error: err }); - return next(err); - } - // Call back error if object metadata could not be obtained - if (!objectMD) { - const err = decodedVidResult ? errors.NoSuchVersion : errors.NoSuchKey; - log.trace('error no object metadata found', { method: METHOD, error: err }); - return next(err, bucketMD); - } - // If object metadata is delete marker, - // call back NoSuchKey or MethodNotAllowed depending on specifying versionId - if (objectMD.isDeleteMarker) { - let err = errorInstances.NoSuchKey; - if (decodedVidResult) { - err = errorInstances.MethodNotAllowed; + return mdUtils.standardMetadataValidateBucketAndObj( + mdValueParams, + request.actionImplicitDenies, + log, + (err, bucketMD, objectMD) => { + if (err) { + log.trace('request authorization failed', { method: METHOD, error: err }); + return next(err); } - log.trace('version is a delete marker', { method: METHOD, error: err }); - return next(err, bucketMD, objectMD); - } - log.debug('acquired the object metadata.', { - 'method': METHOD, - }); - return next(null, bucketMD, objectMD); - }); + // Call back error if object metadata could not be obtained + if (!objectMD) { + const err = decodedVidResult ? errors.NoSuchVersion : errors.NoSuchKey; + log.trace('error no object metadata found', { method: METHOD, error: err }); + return next(err, bucketMD); + } + // If object metadata is delete marker, + // call back NoSuchKey or MethodNotAllowed depending on specifying versionId + if (objectMD.isDeleteMarker) { + let err = errorInstances.NoSuchKey; + if (decodedVidResult) { + err = errorInstances.MethodNotAllowed; + } + log.trace('version is a delete marker', { method: METHOD, error: err }); + return next(err, bucketMD, objectMD); + } + log.debug('acquired the object metadata.', { + method: METHOD, + }); + return next(null, bucketMD, objectMD); + }, + ); }, // generate restore param obj from xml of request body and check tier validity @@ -118,39 +122,47 @@ function objectRestore(metadata, mdUtils, userInfo, request, log, callback) { }, // start restore process function startRestore(bucketMD, objectMD, restoreInfo, next) { - return coldStorage.startRestore(objectMD, restoreInfo, log, - (err, _isObjectRestored) => { - isObjectRestored = _isObjectRestored; - return next(err, bucketMD, objectMD); - }); + return coldStorage.startRestore(objectMD, restoreInfo, log, (err, _isObjectRestored) => { + isObjectRestored = _isObjectRestored; + return next(err, bucketMD, objectMD); + }); }, function evaluateQuotas(bucketMD, objectMD, next) { if (isObjectRestored) { return next(null, bucketMD, objectMD); } - const actions = Array.isArray(mdValueParams.requestType) ? - mdValueParams.requestType : [mdValueParams.requestType]; + const actions = Array.isArray(mdValueParams.requestType) + ? mdValueParams.requestType + : [mdValueParams.requestType]; const bytes = processBytesToWrite(request.apiMethod, bucketMD, mdValueParams.versionId, 0, objectMD); - return validateQuotas(request, bucketMD, request.accountQuotas, actions, request.apiMethod, bytes, - false, log, err => next(err, bucketMD, objectMD)); + return validateQuotas( + request, + bucketMD, + request.accountQuotas, + actions, + request.apiMethod, + bytes, + false, + log, + err => next(err, bucketMD, objectMD), + ); }, function updateObjectMD(bucketMD, objectMD, next) { const params = objectMD.versionId ? { versionId: objectMD.versionId } : {}; - metadata.putObjectMD(bucketMD.getName(), objectKey, objectMD, params, - log, err => next(err, bucketMD, objectMD)); + metadata.putObjectMD(bucketMD.getName(), objectKey, objectMD, params, log, err => + next(err, bucketMD, objectMD), + ); }, ], (err, bucketMD) => { // generate CORS response header const responseHeaders = collectCorsHeaders(request.headers.origin, request.method, bucketMD); if (err) { - log.trace('error processing request', - { - method: METHOD, - error: err, - }); - monitoring.promMetrics( - 'POST', bucketName, err.code, 'restoreObject'); + log.trace('error processing request', { + method: METHOD, + error: err, + }); + monitoring.promMetrics('POST', bucketName, err.code, 'restoreObject'); return callback(err, err.code, responseHeaders); } pushMetric('restoreObject', log, { @@ -158,15 +170,13 @@ function objectRestore(metadata, mdUtils, userInfo, request, log, callback) { bucket: bucketName, }); if (isObjectRestored) { - monitoring.promMetrics( - 'POST', bucketName, '200', 'restoreObject'); + monitoring.promMetrics('POST', bucketName, '200', 'restoreObject'); return callback(null, 200, responseHeaders); } - monitoring.promMetrics( - 'POST', bucketName, '202', 'restoreObject'); + monitoring.promMetrics('POST', bucketName, '202', 'restoreObject'); return callback(null, 202, responseHeaders); - }); + }, + ); } - module.exports = objectRestore; diff --git a/lib/api/apiUtils/object/parseCopySource.js b/lib/api/apiUtils/object/parseCopySource.js index a28770ded3..27a262dbc0 100644 --- a/lib/api/apiUtils/object/parseCopySource.js +++ b/lib/api/apiUtils/object/parseCopySource.js @@ -26,8 +26,7 @@ function parseCopySource(apiMethod, copySourceHeader) { // Pull the source bucket and source object separated by / const sourceBucket = source.slice(0, slashSeparator); const sourceObject = source.slice(slashSeparator + 1); - const sourceVersionId = - decodeVersionId(query ? querystring.parse(query) : undefined); + const sourceVersionId = decodeVersionId(query ? querystring.parse(query) : undefined); if (sourceVersionId instanceof Error) { const err = sourceVersionId; return { parsingError: err }; diff --git a/lib/api/apiUtils/object/partInfo.js b/lib/api/apiUtils/object/partInfo.js index c8715d3628..3480bd0090 100644 --- a/lib/api/apiUtils/object/partInfo.js +++ b/lib/api/apiUtils/object/partInfo.js @@ -6,8 +6,7 @@ */ function getPartNumber(query) { if (query && query.partNumber !== undefined) { - return Number.isNaN(query.partNumber) ? - 0 : Number.parseInt(query.partNumber, 10); + return Number.isNaN(query.partNumber) ? 0 : Number.parseInt(query.partNumber, 10); } return undefined; } @@ -21,14 +20,12 @@ function getPartNumber(query) { function getPartSize(objMD, partNumber) { let size; let locationPartNumber; - if (partNumber && objMD && objMD.location - && objMD.location.length >= partNumber) { + if (partNumber && objMD && objMD.location && objMD.location.length >= partNumber) { const locations = []; for (let i = 0; i < objMD.location.length; i++) { const { dataStoreETag } = objMD.location[i]; if (dataStoreETag) { - locationPartNumber = - Number.parseInt(dataStoreETag.split(':')[0], 10); + locationPartNumber = Number.parseInt(dataStoreETag.split(':')[0], 10); } else { /** * Location objects prior to GA7.1 do not include the diff --git a/lib/api/apiUtils/object/prepareStream.js b/lib/api/apiUtils/object/prepareStream.js index 493e25f185..e1e7d8b49a 100644 --- a/lib/api/apiUtils/object/prepareStream.js +++ b/lib/api/apiUtils/object/prepareStream.js @@ -14,8 +14,7 @@ const TrailingChecksumTransform = require('../../../auth/streamingV4/trailingChe * the type of request requires them */ function prepareStream(stream, streamingV4Params, log, errCb) { - if (stream.headers['x-amz-content-sha256'] === - 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD') { + if (stream.headers['x-amz-content-sha256'] === 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD') { if (typeof streamingV4Params !== 'object') { // this might happen if the user provided a valid V2 // Authentication header, while the chunked upload method diff --git a/lib/api/apiUtils/object/setPartRanges.js b/lib/api/apiUtils/object/setPartRanges.js index 9a2587d1cf..f36a1273c6 100644 --- a/lib/api/apiUtils/object/setPartRanges.js +++ b/lib/api/apiUtils/object/setPartRanges.js @@ -28,8 +28,7 @@ function setPartRanges(dataLocations, outerRange) { // be allowed, so not an issue that size not modified here. if (dataLocations[0].size) { const partSize = parseInt(dataLocations[0].size, 10); - soleLocation.size = - Math.min(partSize, end - begin + 1).toString(); + soleLocation.size = Math.min(partSize, end - begin + 1).toString(); } parsedLocations.push(soleLocation); return parsedLocations; @@ -73,8 +72,7 @@ function setPartRanges(dataLocations, outerRange) { // Use full remaining part if remaining partSize is less // than byte range we need to satisfy. Or use byte range // we need to satisfy taking into account any startOffset - const endPart = Math.min(partSize - 1, - max - total + startOffset - 1); + const endPart = Math.min(partSize - 1, max - total + startOffset - 1); partWithRange.range = [startOffset, endPart]; // modify size to be stored for object put part copy partWithRange.size = (endPart - startOffset + 1).toString(); diff --git a/lib/api/apiUtils/object/setUpCopyLocator.js b/lib/api/apiUtils/object/setUpCopyLocator.js index a232fa3620..5b6cef17e7 100644 --- a/lib/api/apiUtils/object/setUpCopyLocator.js +++ b/lib/api/apiUtils/object/setUpCopyLocator.js @@ -1,8 +1,5 @@ const { errors, errorInstances } = require('arsenal'); -const { - parseRangeSpec, - parseRange, -} = require('arsenal').network.http.utils; +const { parseRangeSpec, parseRange } = require('arsenal').network.http.utils; const constants = require('../../../../constants'); const setPartRanges = require('./setPartRanges'); @@ -15,7 +12,8 @@ const setPartRanges = require('./setPartRanges'); function parseRangeHeader(header) { const { error } = parseRangeSpec(header); if (error) { - const description = 'The x-amz-copy-source-range value must be ' + + const description = + 'The x-amz-copy-source-range value must be ' + 'of the form bytes=first-last where first and last are the ' + 'zero-based offsets of the first and last bytes to copy'; return error.customizeDescription(description); @@ -42,21 +40,17 @@ function setUpCopyLocator(sourceObjMD, rangeHeader, log) { // To provide for backwards compatibility before // md-model-version 2, need to handle cases where // objMD.location is just a string - dataLocator = Array.isArray(sourceObjMD.location) ? - sourceObjMD.location : [{ key: sourceObjMD.location }]; + dataLocator = Array.isArray(sourceObjMD.location) ? sourceObjMD.location : [{ key: sourceObjMD.location }]; } if (sourceObjMD['x-amz-server-side-encryption']) { for (let i = 0; i < dataLocator.length; i++) { - dataLocator[i].masterKeyId = - sourceObjMD['x-amz-server-side-encryption-aws-kms-key-id']; - dataLocator[i].algorithm = - sourceObjMD['x-amz-server-side-encryption']; + dataLocator[i].masterKeyId = sourceObjMD['x-amz-server-side-encryption-aws-kms-key-id']; + dataLocator[i].algorithm = sourceObjMD['x-amz-server-side-encryption']; } } - const sourceSize = - parseInt(sourceObjMD['content-length'], 10); + const sourceSize = parseInt(sourceObjMD['content-length'], 10); let copyObjectSize = sourceSize; if (rangeHeader) { const rangeHeaderError = parseRangeHeader(rangeHeader); @@ -70,15 +64,18 @@ function setUpCopyLocator(sourceObjMD, rangeHeader, log) { // If have a data model before version 2, cannot // support get range copy (do not have size // stored with data locations) - if ((range && dataLocator.length >= 1) && - (dataLocator[0].start === undefined - || dataLocator[0].size === undefined)) { - log.trace('data model before version 2 so ' + - 'cannot support get range copy part'); - return { error: errorInstances.NotImplemented - .customizeDescription('Stored object ' + - 'has legacy data storage model so does' + - ' not support range headers on copy part'), + if ( + range && + dataLocator.length >= 1 && + (dataLocator[0].start === undefined || dataLocator[0].size === undefined) + ) { + log.trace('data model before version 2 so ' + 'cannot support get range copy part'); + return { + error: errorInstances.NotImplemented.customizeDescription( + 'Stored object ' + + 'has legacy data storage model so does' + + ' not support range headers on copy part', + ), }; } if (range) { @@ -87,8 +84,7 @@ function setUpCopyLocator(sourceObjMD, rangeHeader, log) { } } if (copyObjectSize > constants.maximumAllowedPartSize) { - log.debug('copy part size too large', { sourceSize, rangeHeader, - copyObjectSize }); + log.debug('copy part size too large', { sourceSize, rangeHeader, copyObjectSize }); return { error: errors.EntityTooLarge }; } return { dataLocator, copyObjectSize }; diff --git a/lib/api/apiUtils/object/sseHeaders.js b/lib/api/apiUtils/object/sseHeaders.js index 8ed85a828e..6fdc60aee0 100644 --- a/lib/api/apiUtils/object/sseHeaders.js +++ b/lib/api/apiUtils/object/sseHeaders.js @@ -7,8 +7,9 @@ function setSSEHeaders(headers, algo, kmsKey) { headers['x-amz-server-side-encryption'] = algo; if (kmsKey && algo === 'aws:kms') { // eslint-disable-next-line no-param-reassign - headers['x-amz-server-side-encryption-aws-kms-key-id'] = - config.kmsHideScalityArn ? getKeyIdFromArn(kmsKey) : kmsKey; + headers['x-amz-server-side-encryption-aws-kms-key-id'] = config.kmsHideScalityArn + ? getKeyIdFromArn(kmsKey) + : kmsKey; } } } diff --git a/lib/api/apiUtils/object/storeObject.js b/lib/api/apiUtils/object/storeObject.js index 8beea03ecb..e01e899750 100644 --- a/lib/api/apiUtils/object/storeObject.js +++ b/lib/api/apiUtils/object/storeObject.js @@ -55,8 +55,7 @@ function checkHashMatchMD5(stream, hashedStream, dataRetrievalInfo, log, cb) { * @param {function} cb - callback containing result for the next task * @return {undefined} */ -function dataStore(objectContext, cipherBundle, stream, size, - streamingV4Params, backendInfo, log, cb) { +function dataStore(objectContext, cipherBundle, stream, size, streamingV4Params, backendInfo, log, cb) { const cbOnce = jsutil.once(cb); const dataStreamTmp = prepareStream(stream, streamingV4Params, log, cbOnce); if (!dataStreamTmp) { @@ -64,7 +63,12 @@ function dataStore(objectContext, cipherBundle, stream, size, } const dataStream = stripTrailingChecksumStream(dataStreamTmp, log, cbOnce); return data.put( - cipherBundle, dataStream, size, objectContext, backendInfo, log, + cipherBundle, + dataStream, + size, + objectContext, + backendInfo, + log, (err, dataRetrievalInfo, hashedStream) => { if (err) { log.error('error in datastore', { @@ -81,9 +85,9 @@ function dataStore(objectContext, cipherBundle, stream, size, log.trace('dataStore: backend stored key', { dataRetrievalInfo, }); - return checkHashMatchMD5(stream, hashedStream, - dataRetrievalInfo, log, cbOnce); - }); + return checkHashMatchMD5(stream, hashedStream, dataRetrievalInfo, log, cbOnce); + }, + ); } module.exports = { diff --git a/lib/api/apiUtils/object/validateChecksumHeaders.js b/lib/api/apiUtils/object/validateChecksumHeaders.js index d2a50395a3..9c079daaea 100644 --- a/lib/api/apiUtils/object/validateChecksumHeaders.js +++ b/lib/api/apiUtils/object/validateChecksumHeaders.js @@ -5,8 +5,10 @@ const { unsupportedSignatureChecksums, supportedSignatureChecksums } = require(' function validateChecksumHeaders(headers) { // If the x-amz-trailer header is present the request is using one of the // trailing checksum algorithms, which are not supported. - if (headers['x-amz-trailer'] !== undefined && - headers['x-amz-content-sha256'] !== 'STREAMING-UNSIGNED-PAYLOAD-TRAILER') { + if ( + headers['x-amz-trailer'] !== undefined && + headers['x-amz-content-sha256'] !== 'STREAMING-UNSIGNED-PAYLOAD-TRAILER' + ) { return errorInstances.BadRequest.customizeDescription('signed trailing checksum is not supported'); } diff --git a/lib/api/apiUtils/object/versioning.js b/lib/api/apiUtils/object/versioning.js index 418224a4a3..1ece12c652 100644 --- a/lib/api/apiUtils/object/versioning.js +++ b/lib/api/apiUtils/object/versioning.js @@ -10,8 +10,7 @@ const { scaledMsPerDay } = config.getTimeOptions(); const versionIdUtils = versioning.VersionID; // Use Arsenal function to generate a version ID used internally by metadata // for null versions that are created before bucket versioning is configured -const nonVersionedObjId = - versionIdUtils.getInfVid(config.replicationGroupId); +const nonVersionedObjId = versionIdUtils.getInfVid(config.replicationGroupId); /** decodeVID - decode the version id * @param {string} versionId - version ID @@ -101,25 +100,24 @@ function _storeNullVersionMD(bucketName, objKey, nullVersionId, objMD, log, cb) nullVersionMD.originOp = 's3:StoreNullVersion'; metadata.putObjectMD(bucketName, objKey, nullVersionMD, { versionId }, log, err => { if (err) { - log.debug('error from metadata storing null version as new version', - { error: err }); + log.debug('error from metadata storing null version as new version', { error: err }); } - + cb(err); }); } /** check existence and get location of null version data for deletion -* @param {string} bucketName - name of bucket -* @param {string} objKey - name of object key -* @param {object} options - metadata options for getting object MD -* @param {string} options.versionId - version to get from metadata -* @param {object} mst - info about the master version -* @param {string} mst.versionId - the master version's version id -* @param {RequestLogger} log - logger instanceof -* @param {function} cb - callback -* @return {undefined} - and call callback with (err, dataToDelete) -*/ + * @param {string} bucketName - name of bucket + * @param {string} objKey - name of object key + * @param {object} options - metadata options for getting object MD + * @param {string} options.versionId - version to get from metadata + * @param {object} mst - info about the master version + * @param {string} mst.versionId - the master version's version id + * @param {RequestLogger} log - logger instanceof + * @param {function} cb - callback + * @return {undefined} - and call callback with (err, dataToDelete) + */ function _prepareNullVersionDeletion(bucketName, objKey, options, mst, log, cb) { const nullOptions = {}; if (!options.deleteData) { @@ -135,38 +133,40 @@ function _prepareNullVersionDeletion(bucketName, objKey, options, mst, log, cb) // PUT via this option nullOptions.deleteNullKey = true; } - return metadata.getObjectMD(bucketName, objKey, options, log, - (err, versionMD) => { - if (err) { - // the null key may not exist, hence it's a normal - // situation to have a NoSuchKey error, in which case - // there is nothing to delete - if (err.is.NoSuchKey) { - log.debug('null version does not exist', { - method: '_prepareNullVersionDeletion', - }); - } else { - log.warn('could not get null version metadata', { - error: err, - method: '_prepareNullVersionDeletion', - }); - } - return cb(err); - } - if (versionMD.location) { - const dataToDelete = Array.isArray(versionMD.location) ? - versionMD.location : [versionMD.location]; - nullOptions.dataToDelete = dataToDelete; + return metadata.getObjectMD(bucketName, objKey, options, log, (err, versionMD) => { + if (err) { + // the null key may not exist, hence it's a normal + // situation to have a NoSuchKey error, in which case + // there is nothing to delete + if (err.is.NoSuchKey) { + log.debug('null version does not exist', { + method: '_prepareNullVersionDeletion', + }); + } else { + log.warn('could not get null version metadata', { + error: err, + method: '_prepareNullVersionDeletion', + }); } - return cb(null, nullOptions); - }); + return cb(err); + } + if (versionMD.location) { + const dataToDelete = Array.isArray(versionMD.location) ? versionMD.location : [versionMD.location]; + nullOptions.dataToDelete = dataToDelete; + } + return cb(null, nullOptions); + }); } function _deleteNullVersionMD(bucketName, objKey, options, log, cb) { return metadata.deleteObjectMD(bucketName, objKey, options, log, err => { if (err) { - log.warn('metadata error deleting null versioned key', - { bucketName, objKey, error: err, method: '_deleteNullVersionMD' }); + log.warn('metadata error deleting null versioned key', { + bucketName, + objKey, + error: err, + method: '_deleteNullVersionMD', + }); } return cb(err); }); @@ -193,7 +193,7 @@ function _deleteNullVersionMD(bucketName, objKey, options, log, cb) { version key, if needed */ function processVersioningState(mst, vstat, nullVersionCompatMode) { - const versioningSuspended = (vstat === 'Suspended'); + const versioningSuspended = vstat === 'Suspended'; const masterIsNull = mst.exists && (mst.isNull || !mst.versionId); if (versioningSuspended) { @@ -244,7 +244,7 @@ function processVersioningState(mst, vstat, nullVersionCompatMode) { if (masterIsNull) { // if master is a null version or a non-versioned key, // copy it to a new null key - const nullVersionId = (mst.isNull && mst.versionId) ? mst.versionId : nonVersionedObjId; + const nullVersionId = mst.isNull && mst.versionId ? mst.versionId : nonVersionedObjId; if (nullVersionCompatMode) { options.extraMD = { nullVersionId, @@ -311,8 +311,7 @@ function getMasterState(objMD) { }; if (objMD.location) { - mst.objLocation = Array.isArray(objMD.location) ? - objMD.location : [objMD.location]; + mst.objLocation = Array.isArray(objMD.location) ? objMD.location : [objMD.location]; } return mst; @@ -332,8 +331,7 @@ function getMasterState(objMD) { * options.versioning - (true/undefined) metadata instruction to create new ver * options.isNull - (true/undefined) whether new version is null or not */ -function versioningPreprocessing(bucketName, bucketMD, objectKey, objMD, - log, callback) { +function versioningPreprocessing(bucketName, bucketMD, objectKey, objMD, log, callback) { const mst = getMasterState(objMD); const vCfg = bucketMD.getVersioningConfiguration(); @@ -342,50 +340,57 @@ function versioningPreprocessing(bucketName, bucketMD, objectKey, objMD, return process.nextTick(callback, null, options); } - const { options, nullVersionId, delOptions } = - processVersioningState(mst, vCfg.Status, config.nullVersionCompatMode); - - return async.series([ - function storeNullVersionMD(next) { - if (!nullVersionId) { - return process.nextTick(next); - } + const { options, nullVersionId, delOptions } = processVersioningState( + mst, + vCfg.Status, + config.nullVersionCompatMode, + ); + + return async.series( + [ + function storeNullVersionMD(next) { + if (!nullVersionId) { + return process.nextTick(next); + } - options.nullVersionId = nullVersionId; - return _storeNullVersionMD(bucketName, objectKey, nullVersionId, objMD, log, next); - }, - function prepareNullVersionDeletion(next) { - if (!delOptions) { - return process.nextTick(next); - } - return _prepareNullVersionDeletion( - bucketName, objectKey, delOptions, mst, log, - (err, nullOptions) => { + options.nullVersionId = nullVersionId; + return _storeNullVersionMD(bucketName, objectKey, nullVersionId, objMD, log, next); + }, + function prepareNullVersionDeletion(next) { + if (!delOptions) { + return process.nextTick(next); + } + return _prepareNullVersionDeletion(bucketName, objectKey, delOptions, mst, log, (err, nullOptions) => { if (err) { return next(err); } Object.assign(options, nullOptions); return next(); }); - }, - function deleteNullVersionMD(next) { - if (delOptions && - delOptions.versionId && - delOptions.versionId !== 'null') { - // backward-compat: delete old null versioned key - return _deleteNullVersionMD( - bucketName, objectKey, { versionId: delOptions.versionId, overheadField }, log, next); + }, + function deleteNullVersionMD(next) { + if (delOptions && delOptions.versionId && delOptions.versionId !== 'null') { + // backward-compat: delete old null versioned key + return _deleteNullVersionMD( + bucketName, + objectKey, + { versionId: delOptions.versionId, overheadField }, + log, + next, + ); + } + return process.nextTick(next); + }, + ], + err => { + // it's possible there was a prior request that deleted the + // null version, so proceed with putting a new version + if (err && err.is.NoSuchKey) { + return callback(null, options); } - return process.nextTick(next); + return callback(err, options); }, - ], err => { - // it's possible there was a prior request that deleted the - // null version, so proceed with putting a new version - if (err && err.is.NoSuchKey) { - return callback(null, options); - } - return callback(err, options); - }); + ); } /** Return options to pass to Metadata layer for version-specific @@ -545,7 +550,7 @@ function overwritingVersioning(objMD, metadataStoreParams) { restoreRequestedAt: objMD.archive?.restoreRequestedAt, restoreRequestedDays: objMD.archive?.restoreRequestedDays, restoreCompletedAt: new Date(now), - restoreWillExpireAt: new Date(now + (days * scaledMsPerDay)), + restoreWillExpireAt: new Date(now + days * scaledMsPerDay), }; /* eslint-enable no-param-reassign */ diff --git a/lib/api/apiUtils/object/websiteServing.js b/lib/api/apiUtils/object/websiteServing.js index d5f23f15ba..cf5a25f842 100644 --- a/lib/api/apiUtils/object/websiteServing.js +++ b/lib/api/apiUtils/object/websiteServing.js @@ -22,10 +22,8 @@ function findRoutingRule(routingRules, key, errCode) { // no error condition, will have match on first rule even if later // there is more specific rule with error condition. for (let i = 0; i < routingRules.length; i++) { - const prefixFromRule = - routingRules[i].getCondition().keyPrefixEquals; - const errorCodeFromRule = - routingRules[i].getCondition().httpErrorCodeReturnedEquals; + const prefixFromRule = routingRules[i].getCondition().keyPrefixEquals; + const errorCodeFromRule = routingRules[i].getCondition().httpErrorCodeReturnedEquals; if (prefixFromRule !== undefined) { if (!key.startsWith(prefixFromRule)) { // no key match, move on @@ -34,8 +32,7 @@ function findRoutingRule(routingRules, key, errCode) { // add the prefixFromRule to the redirect info // so we can replaceKeyPrefixWith if that is part of redirect // rule - const redirectInfo = Object.assign({ prefixFromRule }, - routingRules[i].getRedirect()); + const redirectInfo = Object.assign({ prefixFromRule }, routingRules[i].getRedirect()); // have key match so check error code match if (errorCodeFromRule !== undefined) { if (errCode === errorCodeFromRule) { @@ -51,8 +48,7 @@ function findRoutingRule(routingRules, key, errCode) { // we have an error code condition but no key condition if (errorCodeFromRule !== undefined) { if (errCode === errorCodeFromRule) { - const redirectInfo = Object.assign({}, - routingRules[i].getRedirect()); + const redirectInfo = Object.assign({}, routingRules[i].getRedirect()); return redirectInfo; } continue; @@ -97,8 +93,7 @@ function extractRedirectInfo(location) { * @return {boolean} true if valid, false if not */ function validateWebsiteHeader(header) { - return (!header || header.startsWith('/') || - header.startsWith('http://') || header.startsWith('https://')); + return !header || header.startsWith('/') || header.startsWith('http://') || header.startsWith('https://'); } /** @@ -115,10 +110,10 @@ function appendWebsiteIndexDocument(request, indexDocumentSuffix, force = false) // find index document if "directory" sent in request if (reqObjectKey.endsWith('/')) { request.objectKey += indexDocumentSuffix; - // find index document if no key provided + // find index document if no key provided } else if (reqObjectKey === '') { request.objectKey = indexDocumentSuffix; - // force for redirect 302 on folder without trailing / that has an index + // force for redirect 302 on folder without trailing / that has an index } else if (force) { request.objectKey += `/${indexDocumentSuffix}`; } diff --git a/lib/api/apiUtils/quotas/quotaUtils.js b/lib/api/apiUtils/quotas/quotaUtils.js index 6c967bec94..0e01fc5c9b 100644 --- a/lib/api/apiUtils/quotas/quotaUtils.js +++ b/lib/api/apiUtils/quotas/quotaUtils.js @@ -1,11 +1,7 @@ const async = require('async'); const { errors } = require('arsenal'); const monitoring = require('../../../utilities/monitoringHandler'); -const { - actionNeedQuotaCheckCopy, - actionNeedQuotaCheck, - actionWithDataDeletion, -} = require('arsenal').policies; +const { actionNeedQuotaCheckCopy, actionNeedQuotaCheck, actionWithDataDeletion } = require('arsenal').policies; const { config } = require('../../../Config'); const QuotaService = require('../../../utilization/instance'); @@ -44,7 +40,7 @@ function processBytesToWrite(apiMethod, bucket, versionId, contentLength, objMD, // but it also replaces the target, which decreases storage bytes -= getHotContentLength(destObjMD); } - } else if (!bucket.isVersioningEnabled() || bucket.isVersioningEnabled() && versionId) { + } else if (!bucket.isVersioningEnabled() || (bucket.isVersioningEnabled() && versionId)) { // object is being deleted (non versioned) or hard-deleted (versioned, as indicated by // the `versionId` field) bytes = -getHotContentLength(objMD); @@ -68,8 +64,7 @@ function processBytesToWrite(apiMethod, bucket, versionId, contentLength, objMD, * @returns {boolean} Returns true if the metric is stale, false otherwise. */ function isMetricStale(metric, resourceType, resourceName, action, inflight, log) { - if (metric.date && Date.now() - new Date(metric.date).getTime() > - QuotaService.maxStaleness) { + if (metric.date && Date.now() - new Date(metric.date).getTime() > QuotaService.maxStaleness) { log.warn('Stale metrics from the quota service, allowing the request', { resourceType, resourceName, @@ -110,70 +105,87 @@ function _evaluateQuotas( let bucketQuotaExceeded = false; let accountQuotaExceeded = false; const creationDate = new Date(bucket.getCreationDate()).getTime(); - return async.parallel({ - bucketQuota: parallelDone => { - if (bucketQuota > 0) { - return QuotaService.getUtilizationMetrics('bucket', - `${bucket.getName()}_${creationDate}`, null, { - action, - inflight, - }, (err, bucketMetrics) => { - if (err || inflight < 0) { - return parallelDone(err); - } - if (!isMetricStale(bucketMetrics, 'bucket', bucket.getName(), action, inflight, log) && - BigInt(bucketMetrics.bytesTotal || 0) + BigInt(inflightForCheck || 0) > bucketQuota) { - log.debug('Bucket quota exceeded', { - bucket: bucket.getName(), + return async.parallel( + { + bucketQuota: parallelDone => { + if (bucketQuota > 0) { + return QuotaService.getUtilizationMetrics( + 'bucket', + `${bucket.getName()}_${creationDate}`, + null, + { action, inflight, - quota: bucketQuota, - bytesTotal: bucketMetrics.bytesTotal, - }); - bucketQuotaExceeded = true; - } - return parallelDone(); - }); - } - return parallelDone(); - }, - accountQuota: parallelDone => { - if (accountQuota > 0 && account?.account) { - return QuotaService.getUtilizationMetrics('account', - account.account, null, { - action, - inflight, - }, (err, accountMetrics) => { - if (err || inflight < 0) { - return parallelDone(err); - } - // Metrics are served as BigInt strings - if (!isMetricStale(accountMetrics, 'account', account.account, action, inflight, log) && - BigInt(accountMetrics.bytesTotal || 0) + BigInt(inflightForCheck || 0) > accountQuota) { - log.debug('Account quota exceeded', { - accountId: account.account, + }, + (err, bucketMetrics) => { + if (err || inflight < 0) { + return parallelDone(err); + } + if ( + !isMetricStale(bucketMetrics, 'bucket', bucket.getName(), action, inflight, log) && + BigInt(bucketMetrics.bytesTotal || 0) + BigInt(inflightForCheck || 0) > bucketQuota + ) { + log.debug('Bucket quota exceeded', { + bucket: bucket.getName(), + action, + inflight, + quota: bucketQuota, + bytesTotal: bucketMetrics.bytesTotal, + }); + bucketQuotaExceeded = true; + } + return parallelDone(); + }, + ); + } + return parallelDone(); + }, + accountQuota: parallelDone => { + if (accountQuota > 0 && account?.account) { + return QuotaService.getUtilizationMetrics( + 'account', + account.account, + null, + { action, inflight, - quota: accountQuota, - bytesTotal: accountMetrics.bytesTotal, - }); - accountQuotaExceeded = true; - } - return parallelDone(); + }, + (err, accountMetrics) => { + if (err || inflight < 0) { + return parallelDone(err); + } + // Metrics are served as BigInt strings + if ( + !isMetricStale(accountMetrics, 'account', account.account, action, inflight, log) && + BigInt(accountMetrics.bytesTotal || 0) + BigInt(inflightForCheck || 0) > accountQuota + ) { + log.debug('Account quota exceeded', { + accountId: account.account, + action, + inflight, + quota: accountQuota, + bytesTotal: accountMetrics.bytesTotal, + }); + accountQuotaExceeded = true; + } + return parallelDone(); + }, + ); + } + return parallelDone(); + }, + }, + err => { + if (err) { + log.warn('Error evaluating quotas', { + error: err.name, + description: err.message, + isInflightDeletion: inflight < 0, }); } - return parallelDone(); + return callback(err, bucketQuotaExceeded, accountQuotaExceeded); }, - }, err => { - if (err) { - log.warn('Error evaluating quotas', { - error: err.name, - description: err.message, - isInflightDeletion: inflight < 0, - }); - } - return callback(err, bucketQuotaExceeded, accountQuotaExceeded); - }); + ); } /** @@ -186,11 +198,13 @@ function _evaluateQuotas( * @returns {undefined} - Returns nothing. */ function monitorQuotaEvaluationDuration(apiMethod, type, code, duration) { - monitoring.quotaEvaluationDuration.labels({ - action: apiMethod, - type, - code, - }).observe(duration / 1e9); + monitoring.quotaEvaluationDuration + .labels({ + action: apiMethod, + type, + code, + }) + .observe(duration / 1e9); } /** @@ -248,76 +262,103 @@ function validateQuotas(request, bucket, account, apiNames, apiMethod, inflight, inflight = 0; } - return async.forEach(apiNames, (apiName, done) => { - // Object copy operations first check the target object, - // meaning the source object, containing the current bytes, - // is checked second. This logic handles these APIs calls by - // ensuring the bytes are positives (i.e., not an object - // replacement). - if (actionNeedQuotaCheckCopy(apiName, apiMethod)) { - // eslint-disable-next-line no-param-reassign - inflight = Math.abs(inflight); - } else if (!actionNeedQuotaCheck[apiName] && !actionWithDataDeletion[apiName]) { - return done(); - } - // When inflights are disabled, the sum of the current utilization metrics - // and the current bytes are compared with the quota. The current bytes - // are not sent to the utilization service. When inflights are enabled, - // the sum of the current utilization metrics only are compared with the - // quota. They include the current inflight bytes sent in the request. - let _inflights = shouldSendInflights ? inflight : undefined; - const inflightForCheck = shouldSendInflights ? 0 : inflight; - return _evaluateQuotas(bucketQuota, accountQuota, bucket, account, _inflights, - inflightForCheck, apiName, log, - (err, _bucketQuotaExceeded, _accountQuotaExceeded) => { - if (err) { - return done(err); - } + return async.forEach( + apiNames, + (apiName, done) => { + // Object copy operations first check the target object, + // meaning the source object, containing the current bytes, + // is checked second. This logic handles these APIs calls by + // ensuring the bytes are positives (i.e., not an object + // replacement). + if (actionNeedQuotaCheckCopy(apiName, apiMethod)) { + // eslint-disable-next-line no-param-reassign + inflight = Math.abs(inflight); + } else if (!actionNeedQuotaCheck[apiName] && !actionWithDataDeletion[apiName]) { + return done(); + } + // When inflights are disabled, the sum of the current utilization metrics + // and the current bytes are compared with the quota. The current bytes + // are not sent to the utilization service. When inflights are enabled, + // the sum of the current utilization metrics only are compared with the + // quota. They include the current inflight bytes sent in the request. + let _inflights = shouldSendInflights ? inflight : undefined; + const inflightForCheck = shouldSendInflights ? 0 : inflight; + return _evaluateQuotas( + bucketQuota, + accountQuota, + bucket, + account, + _inflights, + inflightForCheck, + apiName, + log, + (err, _bucketQuotaExceeded, _accountQuotaExceeded) => { + if (err) { + return done(err); + } - bucketQuotaExceeded = _bucketQuotaExceeded; - accountQuotaExceeded = _accountQuotaExceeded; + bucketQuotaExceeded = _bucketQuotaExceeded; + accountQuotaExceeded = _accountQuotaExceeded; - // Inflights are inverted: in case of cleanup, we just re-issue - // the same API call. - if (_inflights) { - _inflights = -_inflights; - } + // Inflights are inverted: in case of cleanup, we just re-issue + // the same API call. + if (_inflights) { + _inflights = -_inflights; + } - request.finalizerHooks.push((errorFromAPI, _done) => { - const code = (bucketQuotaExceeded || accountQuotaExceeded) ? 429 : 200; - const quotaCleanUpStartTime = process.hrtime.bigint(); - // Quotas are cleaned only in case of error in the API - async.waterfall([ - cb => { - if (errorFromAPI) { - return _evaluateQuotas(bucketQuota, accountQuota, bucket, account, _inflights, - null, apiName, log, cb); - } - return cb(); - }, - ], () => { - monitorQuotaEvaluationDuration(apiMethod, type, code, quotaEvaluationDuration + - Number(process.hrtime.bigint() - quotaCleanUpStartTime)); - return _done(); + request.finalizerHooks.push((errorFromAPI, _done) => { + const code = bucketQuotaExceeded || accountQuotaExceeded ? 429 : 200; + const quotaCleanUpStartTime = process.hrtime.bigint(); + // Quotas are cleaned only in case of error in the API + async.waterfall( + [ + cb => { + if (errorFromAPI) { + return _evaluateQuotas( + bucketQuota, + accountQuota, + bucket, + account, + _inflights, + null, + apiName, + log, + cb, + ); + } + return cb(); + }, + ], + () => { + monitorQuotaEvaluationDuration( + apiMethod, + type, + code, + quotaEvaluationDuration + Number(process.hrtime.bigint() - quotaCleanUpStartTime), + ); + return _done(); + }, + ); }); - }); - return done(); - }); - }, err => { - quotaEvaluationDuration = Number(process.hrtime.bigint() - requestStartTime); - if (err) { - log.warn('Error getting metrics from the quota service, allowing the request', { - error: err.name, - description: err.message, - }); - } - if (!actionWithDataDeletion[apiMethod] && - (bucketQuotaExceeded || accountQuotaExceeded)) { - return callback(errors.QuotaExceeded); - } - return callback(); - }); + return done(); + }, + ); + }, + err => { + quotaEvaluationDuration = Number(process.hrtime.bigint() - requestStartTime); + if (err) { + log.warn('Error getting metrics from the quota service, allowing the request', { + error: err.name, + description: err.message, + }); + } + if (!actionWithDataDeletion[apiMethod] && (bucketQuotaExceeded || accountQuotaExceeded)) { + return callback(errors.QuotaExceeded); + } + return callback(); + }, + ); } module.exports = { diff --git a/lib/api/apiUtils/rateLimit/cleanup.js b/lib/api/apiUtils/rateLimit/cleanup.js index 3a54fbcfaa..081c047c5b 100644 --- a/lib/api/apiUtils/rateLimit/cleanup.js +++ b/lib/api/apiUtils/rateLimit/cleanup.js @@ -16,7 +16,6 @@ function cleanupJob(log, options = {}) { } } - /** * Start periodic cleanup of expired rate limit counters and cached configs * diff --git a/lib/api/apiUtils/rateLimit/client.js b/lib/api/apiUtils/rateLimit/client.js index 203643fc0f..a41b3afbf0 100644 --- a/lib/api/apiUtils/rateLimit/client.js +++ b/lib/api/apiUtils/rateLimit/client.js @@ -50,22 +50,15 @@ class RateLimitClient { const key = `ratelimit:${resourceClass}:${resourceId}:${measure}:emptyAt`; const now = Date.now(); - this.redis.grantTokens( - key, - requested, - interval, - burstCapacity, - now, - (err, result) => { - if (err) { - return cb(err); - } - - // Result is number of tokens granted (0 if denied, partial if limited) - const granted = parseInt(result, 10); - return cb(null, granted); + this.redis.grantTokens(key, requested, interval, burstCapacity, now, (err, result) => { + if (err) { + return cb(err); } - ); + + // Result is number of tokens granted (0 if denied, partial if limited) + const granted = parseInt(result, 10); + return cb(null, granted); + }); } /** @@ -84,5 +77,5 @@ if (config.rateLimiting.enabled) { module.exports = { instance, - RateLimitClient + RateLimitClient, }; diff --git a/lib/api/backbeat/listLifecycleCurrents.js b/lib/api/backbeat/listLifecycleCurrents.js index 4799418847..3dd723c868 100644 --- a/lib/api/backbeat/listLifecycleCurrents.js +++ b/lib/api/backbeat/listLifecycleCurrents.js @@ -4,12 +4,14 @@ const services = require('../../services'); const { standardMetadataValidateBucket } = require('../../metadata/metadataUtils'); const { pushMetric } = require('../../utapi/utilities'); const monitoring = require('../../utilities/monitoringHandler'); -const { getLocationConstraintErrorMessage, processCurrents, - validateMaxScannedEntries } = require('../apiUtils/object/lifecycle'); +const { + getLocationConstraintErrorMessage, + processCurrents, + validateMaxScannedEntries, +} = require('../apiUtils/object/lifecycle'); const { config } = require('../../Config'); -function handleResult(listParams, requestMaxKeys, authInfo, - bucketName, list, isBucketVersioned, log, callback) { +function handleResult(listParams, requestMaxKeys, authInfo, bucketName, list, isBucketVersioned, log, callback) { // eslint-disable-next-line no-param-reassign listParams.maxKeys = requestMaxKeys; const res = processCurrents(bucketName, listParams, isBucketVersioned, list); @@ -35,18 +37,19 @@ function listLifecycleCurrents(authInfo, locationConstraints, request, log, call const bucketName = request.bucketName; log.debug('processing request', { method: 'listLifecycleCurrents' }); - const requestMaxKeys = params['max-keys'] ? - Number.parseInt(params['max-keys'], 10) : 1000; + const requestMaxKeys = params['max-keys'] ? Number.parseInt(params['max-keys'], 10) : 1000; if (Number.isNaN(requestMaxKeys) || requestMaxKeys < 0) { - monitoring.promMetrics( - 'GET', bucketName, 400, 'listLifecycleCurrents'); + monitoring.promMetrics('GET', bucketName, 400, 'listLifecycleCurrents'); return callback(errors.InvalidArgument); } const actualMaxKeys = Math.min(constants.listingHardLimit, requestMaxKeys); const minEntriesToBeScanned = 1; - const { isValid, maxScannedLifecycleListingEntries } = - validateMaxScannedEntries(params, config, minEntriesToBeScanned); + const { isValid, maxScannedLifecycleListingEntries } = validateMaxScannedEntries( + params, + config, + minEntriesToBeScanned, + ); if (!isValid) { monitoring.promMetrics('GET', bucketName, 400, 'listLifecycleCurrents'); return callback(errors.InvalidArgument); @@ -80,8 +83,7 @@ function listLifecycleCurrents(authInfo, locationConstraints, request, log, call return standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => { if (err) { log.debug('error processing request', { method: 'metadataValidateBucket', error: err }); - monitoring.promMetrics( - 'GET', bucketName, err.code, 'listLifecycleCurrents'); + monitoring.promMetrics('GET', bucketName, err.code, 'listLifecycleCurrents'); return callback(err, null); } @@ -93,21 +95,35 @@ function listLifecycleCurrents(authInfo, locationConstraints, request, log, call Contents: [], IsTruncated: false, }; - return handleResult(listParams, requestMaxKeys, authInfo, - bucketName, emptyList, isBucketVersioned, log, callback); + return handleResult( + listParams, + requestMaxKeys, + authInfo, + bucketName, + emptyList, + isBucketVersioned, + log, + callback, + ); } - return services.getLifecycleListing(bucketName, listParams, log, - (err, list) => { + return services.getLifecycleListing(bucketName, listParams, log, (err, list) => { if (err) { log.debug('error processing request', { method: 'services.getLifecycleListing', error: err }); - monitoring.promMetrics( - 'GET', bucketName, err.code, 'listLifecycleCurrents'); + monitoring.promMetrics('GET', bucketName, err.code, 'listLifecycleCurrents'); return callback(err, null); } - return handleResult(listParams, requestMaxKeys, authInfo, - bucketName, list, isBucketVersioned, log, callback); + return handleResult( + listParams, + requestMaxKeys, + authInfo, + bucketName, + list, + isBucketVersioned, + log, + callback, + ); }); }); } diff --git a/lib/api/backbeat/listLifecycleOrphanDeleteMarkers.js b/lib/api/backbeat/listLifecycleOrphanDeleteMarkers.js index 89cab46088..48762ae3b3 100644 --- a/lib/api/backbeat/listLifecycleOrphanDeleteMarkers.js +++ b/lib/api/backbeat/listLifecycleOrphanDeleteMarkers.js @@ -7,8 +7,7 @@ const monitoring = require('../../utilities/monitoringHandler'); const { processOrphans, validateMaxScannedEntries } = require('../apiUtils/object/lifecycle'); const { config } = require('../../Config'); -function handleResult(listParams, requestMaxKeys, authInfo, - bucketName, list, log, callback) { +function handleResult(listParams, requestMaxKeys, authInfo, bucketName, list, log, callback) { // eslint-disable-next-line no-param-reassign listParams.maxKeys = requestMaxKeys; const res = processOrphans(bucketName, listParams, list); @@ -34,11 +33,9 @@ function listLifecycleOrphanDeleteMarkers(authInfo, locationConstraints, request const bucketName = request.bucketName; log.debug('processing request', { method: 'listLifecycleOrphanDeleteMarkers' }); - const requestMaxKeys = params['max-keys'] ? - Number.parseInt(params['max-keys'], 10) : 1000; + const requestMaxKeys = params['max-keys'] ? Number.parseInt(params['max-keys'], 10) : 1000; if (Number.isNaN(requestMaxKeys) || requestMaxKeys < 0) { - monitoring.promMetrics( - 'GET', bucketName, 400, 'listLifecycleOrphanDeleteMarkers'); + monitoring.promMetrics('GET', bucketName, 400, 'listLifecycleOrphanDeleteMarkers'); return callback(errors.InvalidArgument); } const actualMaxKeys = Math.min(constants.listingHardLimit, requestMaxKeys); @@ -46,8 +43,11 @@ function listLifecycleOrphanDeleteMarkers(authInfo, locationConstraints, request // 3 is required as a minimum because we must scan at least three entries to determine version eligibility. // Two entries representing the master key and the following one representing the non-current version. const minEntriesToBeScanned = 3; - const { isValid, maxScannedLifecycleListingEntries } = - validateMaxScannedEntries(params, config, minEntriesToBeScanned); + const { isValid, maxScannedLifecycleListingEntries } = validateMaxScannedEntries( + params, + config, + minEntriesToBeScanned, + ); if (!isValid) { monitoring.promMetrics('GET', bucketName, 400, 'listLifecycleOrphanDeleteMarkers'); return callback(errors.InvalidArgument); @@ -71,8 +71,7 @@ function listLifecycleOrphanDeleteMarkers(authInfo, locationConstraints, request return standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => { if (err) { log.debug('error processing request', { method: 'metadataValidateBucket', error: err }); - monitoring.promMetrics( - 'GET', bucketName, err.code, 'listLifecycleOrphanDeleteMarkers'); + monitoring.promMetrics('GET', bucketName, err.code, 'listLifecycleOrphanDeleteMarkers'); return callback(err, null); } @@ -80,8 +79,7 @@ function listLifecycleOrphanDeleteMarkers(authInfo, locationConstraints, request const isBucketVersioned = vcfg && (vcfg.Status === 'Enabled' || vcfg.Status === 'Suspended'); if (!isBucketVersioned) { log.debug('bucket is not versioned or suspended'); - return callback(errorInstances.InvalidRequest.customizeDescription( - 'bucket is not versioned'), null); + return callback(errorInstances.InvalidRequest.customizeDescription('bucket is not versioned'), null); } if (!requestMaxKeys) { @@ -89,20 +87,16 @@ function listLifecycleOrphanDeleteMarkers(authInfo, locationConstraints, request Contents: [], IsTruncated: false, }; - return handleResult(listParams, requestMaxKeys, authInfo, - bucketName, emptyList, log, callback); + return handleResult(listParams, requestMaxKeys, authInfo, bucketName, emptyList, log, callback); } - return services.getLifecycleListing(bucketName, listParams, log, - (err, list) => { + return services.getLifecycleListing(bucketName, listParams, log, (err, list) => { if (err) { log.debug('error processing request', { error: err }); - monitoring.promMetrics( - 'GET', bucketName, err.code, 'listLifecycleOrphanDeleteMarkers'); + monitoring.promMetrics('GET', bucketName, err.code, 'listLifecycleOrphanDeleteMarkers'); return callback(err, null); } - return handleResult(listParams, requestMaxKeys, authInfo, - bucketName, list, log, callback); + return handleResult(listParams, requestMaxKeys, authInfo, bucketName, list, log, callback); }); }); } diff --git a/lib/api/bucketDelete.js b/lib/api/bucketDelete.js index 78af94dec4..4f08704692 100644 --- a/lib/api/bucketDelete.js +++ b/lib/api/bucketDelete.js @@ -21,8 +21,7 @@ function bucketDelete(authInfo, request, log, cb) { if (authInfo.isRequesterPublicUser()) { log.debug('operation not available for public user'); - monitoring.promMetrics( - 'DELETE', request.bucketName, 403, 'deleteBucket'); + monitoring.promMetrics('DELETE', request.bucketName, 403, 'deleteBucket'); return cb(errors.AccessDenied); } const bucketName = request.bucketName; @@ -34,35 +33,27 @@ function bucketDelete(authInfo, request, log, cb) { request, }; - return standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, - (err, bucketMD) => { - const corsHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucketMD); + return standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucketMD) => { + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucketMD); + if (err) { + log.debug('error processing request', { method: 'metadataValidateBucket', error: err }); + monitoring.promMetrics('DELETE', bucketName, err.code, 'deleteBucket'); + return cb(err, corsHeaders); + } + log.trace('passed checks', { method: 'metadataValidateBucket' }); + return deleteBucket(authInfo, bucketMD, bucketName, authInfo.getCanonicalID(), request, log, err => { if (err) { - log.debug('error processing request', - { method: 'metadataValidateBucket', error: err }); - monitoring.promMetrics( - 'DELETE', bucketName, err.code, 'deleteBucket'); + monitoring.promMetrics('DELETE', bucketName, err.code, 'deleteBucket'); return cb(err, corsHeaders); } - log.trace('passed checks', - { method: 'metadataValidateBucket' }); - return deleteBucket(authInfo, bucketMD, bucketName, - authInfo.getCanonicalID(), request, log, err => { - if (err) { - monitoring.promMetrics( - 'DELETE', bucketName, err.code, 'deleteBucket'); - return cb(err, corsHeaders); - } - pushMetric('deleteBucket', log, { - authInfo, - bucket: bucketName, - }); - monitoring.promMetrics( - 'DELETE', bucketName, '204', 'deleteBucket'); - return cb(null, corsHeaders); - }); + pushMetric('deleteBucket', log, { + authInfo, + bucket: bucketName, + }); + monitoring.promMetrics('DELETE', bucketName, '204', 'deleteBucket'); + return cb(null, corsHeaders); }); + }); } module.exports = bucketDelete; diff --git a/lib/api/bucketDeleteEncryption.js b/lib/api/bucketDeleteEncryption.js index 35bf57c6c9..9e3a3a6d2f 100644 --- a/lib/api/bucketDeleteEncryption.js +++ b/lib/api/bucketDeleteEncryption.js @@ -25,53 +25,55 @@ function bucketDeleteEncryption(authInfo, request, log, callback) { request, }; - return async.waterfall([ - next => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, next), - (bucket, next) => checkExpectedBucketOwner(request.headers, bucket, log, err => next(err, bucket)), - (bucket, next) => { - const sseConfig = bucket.getServerSideEncryption(); + return async.waterfall( + [ + next => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, next), + (bucket, next) => checkExpectedBucketOwner(request.headers, bucket, log, err => next(err, bucket)), + (bucket, next) => { + const sseConfig = bucket.getServerSideEncryption(); - if (sseConfig === null) { - return next(null, bucket); - } + if (sseConfig === null) { + return next(null, bucket); + } - const { isAccountEncryptionEnabled, masterKeyId, algorithm, cryptoScheme } = sseConfig; + const { isAccountEncryptionEnabled, masterKeyId, algorithm, cryptoScheme } = sseConfig; - let updatedSseConfig = null; + let updatedSseConfig = null; - if (!isAccountEncryptionEnabled && masterKeyId) { - // Keep the encryption configuration as a "cache" to avoid generating a new master key: - // - if the default encryption master key is defined at the bucket level (!isAccountEncryptionEnabled), - // - and if a bucket-level default encryption key is already set. - // This "cache" is implemented by storing the configuration in the bucket metadata - // with mandatory set to false, making sure it remains hidden for `getBucketEncryption` operations. - // There is no need to cache the configuration if the default encryption master key is - // managed at the account level, as the master key id in that case is stored directly in - // the account metadata. - updatedSseConfig = { - mandatory: false, - algorithm, - cryptoScheme, - masterKeyId, - }; - } + if (!isAccountEncryptionEnabled && masterKeyId) { + // Keep the encryption configuration as a "cache" to avoid generating a new master key: + // - if the default encryption master key is defined at the bucket level (!isAccountEncryptionEnabled), + // - and if a bucket-level default encryption key is already set. + // This "cache" is implemented by storing the configuration in the bucket metadata + // with mandatory set to false, making sure it remains hidden for `getBucketEncryption` operations. + // There is no need to cache the configuration if the default encryption master key is + // managed at the account level, as the master key id in that case is stored directly in + // the account metadata. + updatedSseConfig = { + mandatory: false, + algorithm, + cryptoScheme, + masterKeyId, + }; + } - bucket.setServerSideEncryption(updatedSseConfig); - return metadata.updateBucket(bucketName, bucket, log, err => next(err, bucket)); + bucket.setServerSideEncryption(updatedSseConfig); + return metadata.updateBucket(bucketName, bucket, log, err => next(err, bucket)); + }, + ], + (err, bucket) => { + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + if (err) { + log.trace('error processing request', { error: err, method: 'bucketDeleteEncryption' }); + return callback(err, corsHeaders); + } + pushMetric('deleteBucketEncryption', log, { + authInfo, + bucket: bucketName, + }); + return callback(null, corsHeaders); }, - ], - (err, bucket) => { - const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); - if (err) { - log.trace('error processing request', { error: err, method: 'bucketDeleteEncryption' }); - return callback(err, corsHeaders); - } - pushMetric('deleteBucketEncryption', log, { - authInfo, - bucket: bucketName, - }); - return callback(null, corsHeaders); - }); + ); } module.exports = bucketDeleteEncryption; diff --git a/lib/api/bucketDeleteLifecycle.js b/lib/api/bucketDeleteLifecycle.js index 31a4148a74..db1b123454 100644 --- a/lib/api/bucketDeleteLifecycle.js +++ b/lib/api/bucketDeleteLifecycle.js @@ -28,8 +28,7 @@ function bucketDeleteLifecycle(authInfo, request, log, callback) { error: err, method: 'bucketDeleteLifecycle', }); - monitoring.promMetrics( - 'DELETE', bucketName, err.code, 'deleteBucketLifecycle'); + monitoring.promMetrics('DELETE', bucketName, err.code, 'deleteBucketLifecycle'); return callback(err, corsHeaders); } if (!bucket.getLifecycleConfiguration()) { @@ -46,16 +45,14 @@ function bucketDeleteLifecycle(authInfo, request, log, callback) { bucket.setLifecycleConfiguration(null); return metadata.updateBucket(bucketName, bucket, log, err => { if (err) { - monitoring.promMetrics( - 'DELETE', bucketName, err.code, 'deleteBucketLifecycle'); + monitoring.promMetrics('DELETE', bucketName, err.code, 'deleteBucketLifecycle'); return callback(err, corsHeaders); } pushMetric('deleteBucketLifecycle', log, { authInfo, bucket: bucketName, }); - monitoring.promMetrics( - 'DELETE', bucketName, '200', 'deleteBucketLifecycle'); + monitoring.promMetrics('DELETE', bucketName, '200', 'deleteBucketLifecycle'); return callback(null, corsHeaders); }); }); diff --git a/lib/api/bucketDeleteQuota.js b/lib/api/bucketDeleteQuota.js index 849072a45f..6584ce62df 100644 --- a/lib/api/bucketDeleteQuota.js +++ b/lib/api/bucketDeleteQuota.js @@ -25,34 +25,35 @@ function bucketDeleteQuota(authInfo, request, log, callback) { requestType: request.apiMethods || requestType, request, }; - return waterfall([ - next => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, - (err, bucket) => next(err, bucket)), - (bucket, next) => { - bucket.setQuota(0); - metadata.updateBucket(bucket.getName(), bucket, log, err => - next(err, bucket)); - }, - ], (err, bucket) => { - const corsHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); - if (err) { - log.debug('error processing request', { - error: err, - method: 'bucketDeleteQuota' + return waterfall( + [ + next => + standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => + next(err, bucket), + ), + (bucket, next) => { + bucket.setQuota(0); + metadata.updateBucket(bucket.getName(), bucket, log, err => next(err, bucket)); + }, + ], + (err, bucket) => { + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + if (err) { + log.debug('error processing request', { + error: err, + method: 'bucketDeleteQuota', + }); + monitoring.promMetrics('DELETE', bucketName, err.code, 'bucketDeleteQuota'); + return callback(err, err.code, corsHeaders); + } + monitoring.promMetrics('DELETE', bucketName, '204', 'bucketDeleteQuota'); + pushMetric('bucketDeleteQuota', log, { + authInfo, + bucket: bucketName, }); - monitoring.promMetrics('DELETE', bucketName, err.code, - 'bucketDeleteQuota'); - return callback(err, err.code, corsHeaders); - } - monitoring.promMetrics( - 'DELETE', bucketName, '204', 'bucketDeleteQuota'); - pushMetric('bucketDeleteQuota', log, { - authInfo, - bucket: bucketName, - }); - return callback(null, 204, corsHeaders); - }); + return callback(null, 204, corsHeaders); + }, + ); } module.exports = bucketDeleteQuota; diff --git a/lib/api/bucketDeleteReplication.js b/lib/api/bucketDeleteReplication.js index ff9912171e..220284ffcb 100644 --- a/lib/api/bucketDeleteReplication.js +++ b/lib/api/bucketDeleteReplication.js @@ -28,8 +28,7 @@ function bucketDeleteReplication(authInfo, request, log, callback) { error: err, method: 'bucketDeleteReplication', }); - monitoring.promMetrics( - 'DELETE', bucketName, err.code, 'deleteBucketReplication'); + monitoring.promMetrics('DELETE', bucketName, err.code, 'deleteBucketReplication'); return callback(err, corsHeaders); } if (!bucket.getReplicationConfiguration()) { @@ -46,16 +45,14 @@ function bucketDeleteReplication(authInfo, request, log, callback) { bucket.setReplicationConfiguration(null); return metadata.updateBucket(bucketName, bucket, log, err => { if (err) { - monitoring.promMetrics( - 'DELETE', bucketName, err.code, 'deleteBucketReplication'); + monitoring.promMetrics('DELETE', bucketName, err.code, 'deleteBucketReplication'); return callback(err, corsHeaders); } pushMetric('deleteBucketReplication', log, { authInfo, bucket: bucketName, }); - monitoring.promMetrics( - 'DELETE', bucketName, '200', 'deleteBucketReplication'); + monitoring.promMetrics('DELETE', bucketName, '200', 'deleteBucketReplication'); return callback(null, corsHeaders); }); }); diff --git a/lib/api/bucketDeleteTagging.js b/lib/api/bucketDeleteTagging.js index 8997d760ef..0ac35e5082 100644 --- a/lib/api/bucketDeleteTagging.js +++ b/lib/api/bucketDeleteTagging.js @@ -25,38 +25,38 @@ function bucketDeleteTagging(authInfo, request, log, callback) { }; let bucket = null; - return waterfall([ - next => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, - (err, b) => { - if (err) { - return next(err); - } - bucket = b; - bucket.setTags([]); - return next(); - }), - next => metadata.updateBucket(bucket.getName(), bucket, log, next), - ], err => { - const corsHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); - if (err) { - log.error('error processing request', { - error: err, - method: 'deleteBucketTagging', - bucketName + return waterfall( + [ + next => + standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, b) => { + if (err) { + return next(err); + } + bucket = b; + bucket.setTags([]); + return next(); + }), + next => metadata.updateBucket(bucket.getName(), bucket, log, next), + ], + err => { + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + if (err) { + log.error('error processing request', { + error: err, + method: 'deleteBucketTagging', + bucketName, + }); + monitoring.promMetrics('DELETE', bucketName, err.code, 'deleteBucketTagging'); + return callback(err, corsHeaders); + } + pushMetric('deleteBucketTagging', log, { + authInfo, + bucket: bucketName, }); - monitoring.promMetrics('DELETE', bucketName, err.code, - 'deleteBucketTagging'); + monitoring.promMetrics('DELETE', bucketName, '200', 'deleteBucketTagging'); return callback(err, corsHeaders); - } - pushMetric('deleteBucketTagging', log, { - authInfo, - bucket: bucketName, - }); - monitoring.promMetrics( - 'DELETE', bucketName, '200', 'deleteBucketTagging'); - return callback(err, corsHeaders); - }); + }, + ); } module.exports = bucketDeleteTagging; diff --git a/lib/api/bucketGet.js b/lib/api/bucketGet.js index 42689a6d17..9f1e994e5e 100644 --- a/lib/api/bucketGet.js +++ b/lib/api/bucketGet.js @@ -106,7 +106,9 @@ function processVersions(bucketName, listParams, list) { xml.push( '', '', - '', bucketName, '' + '', + bucketName, + '', ); const isTruncated = list.IsTruncated ? 'true' : 'false'; const xmlParams = [ @@ -124,9 +126,8 @@ function processVersions(bucketName, listParams, list) { const escapeXmlFn = listParams.encoding === 'url' ? querystring.escape : escapeForXml; xmlParams.forEach(p => { if (p.value) { - const val = p.tag !== 'NextVersionIdMarker' || p.value === 'null' ? - p.value : - versionIdUtils.encode(p.value); + const val = + p.tag !== 'NextVersionIdMarker' || p.value === 'null' ? p.value : versionIdUtils.encode(p.value); xml.push(`<${p.tag}>${escapeXmlFn(val)}`); } }); @@ -141,9 +142,7 @@ function processVersions(bucketName, listParams, list) { v.IsDeleteMarker ? '' : '', `${objectKey}`, '', - (v.IsNull || v.VersionId === undefined) ? - 'null' - : versionIdUtils.encode(v.VersionId), + v.IsNull || v.VersionId === undefined ? 'null' : versionIdUtils.encode(v.VersionId), '', `${isLatest}`, `${v.LastModified}`, @@ -159,10 +158,7 @@ function processVersions(bucketName, listParams, list) { buildAttributesXml(v, v.userMetadata, listParams.optionalAttributes, xml); } - xml.push( - `${v.StorageClass}`, - v.IsDeleteMarker ? '' : '' - ); + xml.push(`${v.StorageClass}`, v.IsDeleteMarker ? '' : ''); }); list.CommonPrefixes.forEach(item => { @@ -179,7 +175,9 @@ function processMasterVersions(bucketName, listParams, list) { xml.push( '', '', - '', bucketName, '' + '', + bucketName, + '', ); const isTruncated = list.IsTruncated ? 'true' : 'false'; const xmlParams = [ @@ -207,11 +205,13 @@ function processMasterVersions(bucketName, listParams, list) { xml.push(`<${p.tag}>${p.value}`); } else if (p.value || p.tag === 'KeyCount' || p.tag === 'MaxKeys') { xml.push(`<${p.tag}>${escapeXmlFn(p.value)}`); - } else if (p.tag !== 'NextMarker' && - p.tag !== 'EncodingType' && - p.tag !== 'Delimiter' && - p.tag !== 'StartAfter' && - p.tag !== 'NextContinuationToken') { + } else if ( + p.tag !== 'NextMarker' && + p.tag !== 'EncodingType' && + p.tag !== 'Delimiter' && + p.tag !== 'StartAfter' && + p.tag !== 'NextContinuationToken' + ) { xml.push(`<${p.tag}/>`); } }); @@ -228,7 +228,7 @@ function processMasterVersions(bucketName, listParams, list) { `${objectKey}`, `${v.LastModified}`, `"${v.ETag}"`, - `${v.Size}` + `${v.Size}`, ); if (!listParams.v2 || listParams.fetchOwner) { @@ -236,16 +236,13 @@ function processMasterVersions(bucketName, listParams, list) { '', `${v.Owner.ID}`, `${v.Owner.DisplayName}`, - '' + '', ); } buildAttributesXml(v, v.userMetadata, listParams.optionalAttributes, xml); - return xml.push( - `${v.StorageClass}`, - '' - ); + return xml.push(`${v.StorageClass}`, ''); }); list.CommonPrefixes.forEach(item => { xml.push(`${escapeXmlFn(item)}`); @@ -375,9 +372,9 @@ async function bucketGet(authInfo, request, log, callback) { listParams.listingType = 'DelimiterVersions'; delete listParams.marker; listParams.keyMarker = params['key-marker']; - listParams.versionIdMarker = params['version-id-marker'] ? - versionIdUtils.decode(params['version-id-marker']) : - undefined; + listParams.versionIdMarker = params['version-id-marker'] + ? versionIdUtils.decode(params['version-id-marker']) + : undefined; } if (!requestMaxKeys) { const emptyList = { diff --git a/lib/api/bucketGetACL.js b/lib/api/bucketGetACL.js index 0726e06549..7eb47ced9a 100644 --- a/lib/api/bucketGetACL.js +++ b/lib/api/bucketGetACL.js @@ -26,7 +26,6 @@ const monitoring = require('../utilities/monitoringHandler'); */ - /** * bucketGetACL - Return ACL's for bucket * @param {AuthInfo} authInfo - Instance of AuthInfo class with requester's info @@ -56,13 +55,10 @@ function bucketGetACL(authInfo, request, log, callback) { }; standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => { - const corsHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); if (err) { - log.debug('error processing request', - { method: 'bucketGetACL', error: err }); - monitoring.promMetrics( - 'GET', bucketName, err.code, 'getBucketAcl'); + log.debug('error processing request', { method: 'bucketGetACL', error: err }); + monitoring.promMetrics('GET', bucketName, err.code, 'getBucketAcl'); return callback(err, null, corsHeaders); } const bucketACL = bucket.getAcl(); @@ -75,8 +71,7 @@ function bucketGetACL(authInfo, request, log, callback) { }; if (bucketACL.Canned !== '') { - const cannedGrants = aclUtils.handleCannedGrant( - bucketACL.Canned, ownerGrant); + const cannedGrants = aclUtils.handleCannedGrant(bucketACL.Canned, ownerGrant); grantInfo.grants = grantInfo.grants.concat(cannedGrants); const xml = aclUtils.convertToXml(grantInfo); pushMetric('getBucketAcl', log, { @@ -86,19 +81,19 @@ function bucketGetACL(authInfo, request, log, callback) { return callback(null, xml, corsHeaders); } /** - * Build array of all canonicalIDs used in ACLs so duplicates - * will be retained (e.g. if an account has both read and write - * privileges, want to display both and not lose the duplicate - * when receive one dictionary entry back from Vault) - */ + * Build array of all canonicalIDs used in ACLs so duplicates + * will be retained (e.g. if an account has both read and write + * privileges, want to display both and not lose the duplicate + * when receive one dictionary entry back from Vault) + */ const canonicalIDs = aclUtils.getCanonicalIDs(bucketACL); // Build array with grants by URI const uriGrantInfo = aclUtils.getUriGrantInfo(bucketACL); if (canonicalIDs.length === 0) { /** - * If no acl's set by account canonicalID, just add URI - * grants (if any) and return - */ + * If no acl's set by account canonicalID, just add URI + * grants (if any) and return + */ grantInfo.grants = grantInfo.grants.concat(uriGrantInfo); const xml = aclUtils.convertToXml(grantInfo); pushMetric('getBucketAcl', log, { @@ -108,22 +103,18 @@ function bucketGetACL(authInfo, request, log, callback) { return callback(null, xml, corsHeaders); } /** - * If acl's set by account canonicalID, get emails from Vault to serve - * as display names - */ + * If acl's set by account canonicalID, get emails from Vault to serve + * as display names + */ return vault.getEmailAddresses(canonicalIDs, log, (err, emails) => { if (err) { - log.debug('error processing request', - { method: 'vault.getEmailAddresses', error: err }); - monitoring.promMetrics( - 'GET', bucketName, err.code, 'getBucketAcl'); + log.debug('error processing request', { method: 'vault.getEmailAddresses', error: err }); + monitoring.promMetrics('GET', bucketName, err.code, 'getBucketAcl'); return callback(err, null, corsHeaders); } - const individualGrants = - aclUtils.getIndividualGrants(bucketACL, canonicalIDs, emails); + const individualGrants = aclUtils.getIndividualGrants(bucketACL, canonicalIDs, emails); // Add to grantInfo any individual grants and grants by uri - grantInfo.grants = grantInfo.grants - .concat(individualGrants).concat(uriGrantInfo); + grantInfo.grants = grantInfo.grants.concat(individualGrants).concat(uriGrantInfo); // parse info about accounts and owner info to convert to xml const xml = aclUtils.convertToXml(grantInfo); pushMetric('getBucketAcl', log, { diff --git a/lib/api/bucketGetEncryption.js b/lib/api/bucketGetEncryption.js index db5d31432b..a71c3c90f8 100644 --- a/lib/api/bucketGetEncryption.js +++ b/lib/api/bucketGetEncryption.js @@ -28,61 +28,65 @@ function bucketGetEncryption(authInfo, request, log, callback) { request, }; - return async.waterfall([ - next => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, next), - (bucket, next) => checkExpectedBucketOwner(request.headers, bucket, log, err => next(err, bucket)), - (bucket, next) => { - // If sseInfo is present but the `mandatory` flag is not set - // then this info was not created using bucketPutEncryption - // or by using the x-amz-scal-server-side-encryption header at - // bucket creation and should not be returned - const sseInfo = bucket.getServerSideEncryption(); - if (sseInfo === null || !sseInfo.mandatory) { - log.trace('no server side encryption config found', { - bucket: bucketName, - method: 'bucketGetEncryption', - }); - return next(errors.ServerSideEncryptionConfigurationNotFoundError); + return async.waterfall( + [ + next => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, next), + (bucket, next) => checkExpectedBucketOwner(request.headers, bucket, log, err => next(err, bucket)), + (bucket, next) => { + // If sseInfo is present but the `mandatory` flag is not set + // then this info was not created using bucketPutEncryption + // or by using the x-amz-scal-server-side-encryption header at + // bucket creation and should not be returned + const sseInfo = bucket.getServerSideEncryption(); + if (sseInfo === null || !sseInfo.mandatory) { + log.trace('no server side encryption config found', { + bucket: bucketName, + method: 'bucketGetEncryption', + }); + return next(errors.ServerSideEncryptionConfigurationNotFoundError); + } + return next(null, bucket, sseInfo); + }, + ], + (error, bucket, sseInfo) => { + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + if (error) { + return callback(error, corsHeaders); } - return next(null, bucket, sseInfo); - }, - ], - (error, bucket, sseInfo) => { - const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); - if (error) { - return callback(error, corsHeaders); - } - const xml = [ - '', - '', - '', - '', - `${escapeForXml(sseInfo.algorithm)}`, - ]; + const xml = [ + '', + '', + '', + '', + `${escapeForXml(sseInfo.algorithm)}`, + ]; - if (sseInfo.configuredMasterKeyId) { - xml.push(`${escapeForXml( - config.kmsHideScalityArn - ? getKeyIdFromArn(sseInfo.configuredMasterKeyId) - : sseInfo.configuredMasterKeyId - )}`); - } + if (sseInfo.configuredMasterKeyId) { + xml.push( + `${escapeForXml( + config.kmsHideScalityArn + ? getKeyIdFromArn(sseInfo.configuredMasterKeyId) + : sseInfo.configuredMasterKeyId, + )}`, + ); + } - xml.push( - '', - 'false', - '', - '' - ); + xml.push( + '', + 'false', + '', + '', + ); - pushMetric('getBucketEncryption', log, { - authInfo, - bucket: bucketName, - }); + pushMetric('getBucketEncryption', log, { + authInfo, + bucket: bucketName, + }); - return callback(null, xml.join(''), corsHeaders); - }); + return callback(null, xml.join(''), corsHeaders); + }, + ); } module.exports = bucketGetEncryption; diff --git a/lib/api/bucketGetLifecycle.js b/lib/api/bucketGetLifecycle.js index bc2851172c..8a8e5bca72 100644 --- a/lib/api/bucketGetLifecycle.js +++ b/lib/api/bucketGetLifecycle.js @@ -1,6 +1,5 @@ const { errors } = require('arsenal'); -const LifecycleConfiguration = - require('arsenal').models.LifecycleConfiguration; +const LifecycleConfiguration = require('arsenal').models.LifecycleConfiguration; const { standardMetadataValidateBucket } = require('../metadata/metadataUtils'); const { pushMetric } = require('../utapi/utilities'); @@ -31,8 +30,7 @@ function bucketGetLifecycle(authInfo, request, log, callback) { error: err, method: 'bucketGetLifecycle', }); - monitoring.promMetrics( - 'GET', bucketName, err.code, 'getBucketLifecycle'); + monitoring.promMetrics('GET', bucketName, err.code, 'getBucketLifecycle'); return callback(err, null, corsHeaders); } const lifecycleConfig = bucket.getLifecycleConfiguration(); @@ -41,10 +39,8 @@ function bucketGetLifecycle(authInfo, request, log, callback) { error: errors.NoSuchLifecycleConfiguration, method: 'bucketGetLifecycle', }); - monitoring.promMetrics( - 'GET', bucketName, 404, 'getBucketLifecycle'); - return callback(errors.NoSuchLifecycleConfiguration, null, - corsHeaders); + monitoring.promMetrics('GET', bucketName, 404, 'getBucketLifecycle'); + return callback(errors.NoSuchLifecycleConfiguration, null, corsHeaders); } const xml = LifecycleConfiguration.getConfigXml(lifecycleConfig); pushMetric('getBucketLifecycle', log, { diff --git a/lib/api/bucketGetLocation.js b/lib/api/bucketGetLocation.js index 31d8b6dfec..8e4c7db549 100644 --- a/lib/api/bucketGetLocation.js +++ b/lib/api/bucketGetLocation.js @@ -40,7 +40,8 @@ function bucketGetLocation(authInfo, request, log, callback) { // if no locationConstraint provided. locationConstraint = ''; } - const xml = ` + const xml = + ` ` + `${escapeForXml(locationConstraint)}`; pushMetric(METRICS_ACTION, log, { authInfo, bucket: bucketName }); diff --git a/lib/api/bucketGetLogging.js b/lib/api/bucketGetLogging.js index 501caf8942..585d4f460f 100644 --- a/lib/api/bucketGetLogging.js +++ b/lib/api/bucketGetLogging.js @@ -5,7 +5,8 @@ const { waterfall } = require('async'); const { config, serverAccessLogsModes } = require('../Config'); const { errorInstances } = require('arsenal'); -const BucketLoggingStatusNotFoundBody = '\n' + +const BucketLoggingStatusNotFoundBody = + '\n' + ''; function bucketGetLogging(authInfo, request, log, callback) { @@ -23,39 +24,44 @@ function bucketGetLogging(authInfo, request, log, callback) { request, }; - return waterfall([ - next => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => { - if (err) { - return next(err); - } + return waterfall( + [ + next => + standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => { + if (err) { + return next(err); + } - return next(null, bucket); - }), - (bucket, next) => checkExpectedBucketOwner(request.headers, bucket, log, err => { - if (err) { - return next(err); - } + return next(null, bucket); + }), + (bucket, next) => + checkExpectedBucketOwner(request.headers, bucket, log, err => { + if (err) { + return next(err); + } + + return next(null, bucket); + }), + (bucket, next) => { + const bucketLoggingStatus = bucket.getBucketLoggingStatus(); + if (!bucketLoggingStatus) { + return next(null, BucketLoggingStatusNotFoundBody); + } - return next(null, bucket); - }), - (bucket, next) => { - const bucketLoggingStatus = bucket.getBucketLoggingStatus(); - if (!bucketLoggingStatus) { - return next(null, BucketLoggingStatusNotFoundBody); + return next(null, bucketLoggingStatus.toXML()); + }, + ], + (err, body) => { + if (err) { + log.trace('error processing request', { error: err, method: 'bucketGetLogging' }); + monitoring.promMetrics('GET', bucketName, err.code, 'bucketGetLogging'); + return callback(err); } - return next(null, bucketLoggingStatus.toXML()); - } - ], (err, body) => { - if (err) { - log.trace('error processing request', { error: err, method: 'bucketGetLogging' }); - monitoring.promMetrics('GET', bucketName, err.code, 'bucketGetLogging'); - return callback(err); - } - - monitoring.promMetrics('GET', bucketName, '200', 'bucketGetLogging'); - return callback(null, body); - }); + monitoring.promMetrics('GET', bucketName, '200', 'bucketGetLogging'); + return callback(null, body); + }, + ); } module.exports = bucketGetLogging; diff --git a/lib/api/bucketGetObjectLock.js b/lib/api/bucketGetObjectLock.js index a96e7cb4e4..e2aab76c56 100644 --- a/lib/api/bucketGetObjectLock.js +++ b/lib/api/bucketGetObjectLock.js @@ -2,8 +2,7 @@ const { errors } = require('arsenal'); const { standardMetadataValidateBucket } = require('../metadata/metadataUtils'); const { pushMetric } = require('../utapi/utilities'); const collectCorsHeaders = require('../utilities/collectCorsHeaders'); -const ObjectLockConfiguration = - require('arsenal').models.ObjectLockConfiguration; +const ObjectLockConfiguration = require('arsenal').models.ObjectLockConfiguration; // Format of the xml response: /** @@ -53,8 +52,7 @@ function bucketGetObjectLock(authInfo, request, log, callback) { error: errors.ObjectLockConfigurationNotFoundError, method: 'bucketGetObjectLock', }); - return callback(errors.ObjectLockConfigurationNotFoundError, null, - corsHeaders); + return callback(errors.ObjectLockConfigurationNotFoundError, null, corsHeaders); } const xml = ObjectLockConfiguration.getConfigXML(objLockConfig); pushMetric('getBucketObjectLock', log, { diff --git a/lib/api/bucketGetPolicy.js b/lib/api/bucketGetPolicy.js index d6f3b6a576..1f521dc1d9 100644 --- a/lib/api/bucketGetPolicy.js +++ b/lib/api/bucketGetPolicy.js @@ -36,8 +36,7 @@ function bucketGetPolicy(authInfo, request, log, callback) { error: errors.NoSuchBucketPolicy, method: 'bucketGetPolicy', }); - return callback(errors.NoSuchBucketPolicy, null, - corsHeaders); + return callback(errors.NoSuchBucketPolicy, null, corsHeaders); } // TODO: implement Utapi metric support // bucketPolicy needs to be JSON stringified on return for proper diff --git a/lib/api/bucketGetQuota.js b/lib/api/bucketGetQuota.js index 9556e31d4d..963eaecacb 100644 --- a/lib/api/bucketGetQuota.js +++ b/lib/api/bucketGetQuota.js @@ -31,21 +31,15 @@ function bucketGetQuota(authInfo, request, log, callback) { }); return callback(err, null, corsHeaders); } - xml.push( - '', - '', - '', bucket.getName(), '', - ); + xml.push('', '', '', bucket.getName(), ''); const bucketQuota = bucket.getQuota(); if (!bucketQuota) { log.debug('bucket has no quota', { method: 'bucketGetQuota', }); - return callback(errors.NoSuchQuota, null, - corsHeaders); + return callback(errors.NoSuchQuota, null, corsHeaders); } - xml.push('', bucketQuota, '', - ''); + xml.push('', bucketQuota, '', ''); pushMetric('getBucketQuota', log, { authInfo, diff --git a/lib/api/bucketGetRateLimit.js b/lib/api/bucketGetRateLimit.js index 8372e9afd4..48538a604b 100644 --- a/lib/api/bucketGetRateLimit.js +++ b/lib/api/bucketGetRateLimit.js @@ -43,8 +43,7 @@ function bucketGetRateLimit(authInfo, request, log, callback) { error: errors.NoSuchRateLimitConfig, method: 'bucketGetRateLimit', }); - return callback(errors.NoSuchRateLimitConfig, null, - corsHeaders); + return callback(errors.NoSuchRateLimitConfig, null, corsHeaders); } return callback(null, JSON.stringify(rateLimitConfig.getData()), corsHeaders); diff --git a/lib/api/bucketGetReplication.js b/lib/api/bucketGetReplication.js index 8891f96bc1..e1a75a1e20 100644 --- a/lib/api/bucketGetReplication.js +++ b/lib/api/bucketGetReplication.js @@ -2,8 +2,7 @@ const { errors } = require('arsenal'); const { standardMetadataValidateBucket } = require('../metadata/metadataUtils'); const { pushMetric } = require('../utapi/utilities'); -const { getReplicationConfigurationXML } = - require('./apiUtils/bucket/getReplicationConfiguration'); +const { getReplicationConfigurationXML } = require('./apiUtils/bucket/getReplicationConfiguration'); const collectCorsHeaders = require('../utilities/collectCorsHeaders'); const monitoring = require('../utilities/monitoringHandler'); @@ -31,8 +30,7 @@ function bucketGetReplication(authInfo, request, log, callback) { error: err, method: 'bucketGetReplication', }); - monitoring.promMetrics( - 'GET', bucketName, err.code, 'getBucketReplication'); + monitoring.promMetrics('GET', bucketName, err.code, 'getBucketReplication'); return callback(err, null, corsHeaders); } const replicationConfig = bucket.getReplicationConfiguration(); @@ -41,18 +39,15 @@ function bucketGetReplication(authInfo, request, log, callback) { error: errors.ReplicationConfigurationNotFoundError, method: 'bucketGetReplication', }); - monitoring.promMetrics( - 'GET', bucketName, 404, 'getBucketReplication'); - return callback(errors.ReplicationConfigurationNotFoundError, null, - corsHeaders); + monitoring.promMetrics('GET', bucketName, 404, 'getBucketReplication'); + return callback(errors.ReplicationConfigurationNotFoundError, null, corsHeaders); } const xml = getReplicationConfigurationXML(replicationConfig); pushMetric('getBucketReplication', log, { authInfo, bucket: bucketName, }); - monitoring.promMetrics( - 'GET', bucketName, '200', 'getBucketReplication'); + monitoring.promMetrics('GET', bucketName, '200', 'getBucketReplication'); return callback(null, xml, corsHeaders); }); } diff --git a/lib/api/bucketGetTagging.js b/lib/api/bucketGetTagging.js index c31a14a08a..83c4de8e45 100644 --- a/lib/api/bucketGetTagging.js +++ b/lib/api/bucketGetTagging.js @@ -74,44 +74,44 @@ function bucketGetTagging(authInfo, request, log, callback) { let xml = null; let tags = null; - return waterfall([ - next => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, - (err, b) => { - bucket = b; - return next(err); - }), - next => checkExpectedBucketOwner(headers, bucket, log, next), - next => { - tags = bucket.getTags(); - if (!tags || !tags.length) { - log.debug('bucket TagSet does not exist', { + return waterfall( + [ + next => + standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, b) => { + bucket = b; + return next(err); + }), + next => checkExpectedBucketOwner(headers, bucket, log, next), + next => { + tags = bucket.getTags(); + if (!tags || !tags.length) { + log.debug('bucket TagSet does not exist', { + method: 'bucketGetTagging', + }); + return next(errors.NoSuchTagSet); + } + xml = tagsToXml(tags); + return next(); + }, + ], + err => { + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + if (err) { + log.debug('error processing request', { + error: err, method: 'bucketGetTagging', }); - return next(errors.NoSuchTagSet); + monitoring.promMetrics('GET', bucketName, err.code, 'getBucketTagging'); + } else { + pushMetric('getBucketTagging', log, { + authInfo, + bucket: bucketName, + }); + monitoring.promMetrics('GET', bucketName, '200', 'getBucketTagging'); } - xml = tagsToXml(tags); - return next(); - } - ], err => { - const corsHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); - if (err) { - log.debug('error processing request', { - error: err, - method: 'bucketGetTagging' - }); - monitoring.promMetrics('GET', bucketName, err.code, - 'getBucketTagging'); - } else { - pushMetric('getBucketTagging', log, { - authInfo, - bucket: bucketName, - }); - monitoring.promMetrics( - 'GET', bucketName, '200', 'getBucketTagging'); - } - return callback(err, xml, corsHeaders); - }); + return callback(err, xml, corsHeaders); + }, + ); } module.exports = bucketGetTagging; diff --git a/lib/api/bucketGetVersioning.js b/lib/api/bucketGetVersioning.js index f38cc31053..101d0eb8b7 100644 --- a/lib/api/bucketGetVersioning.js +++ b/lib/api/bucketGetVersioning.js @@ -19,9 +19,9 @@ const monitoring = require('../utilities/monitoringHandler'); function convertToXml(versioningConfiguration) { const xml = []; - xml.push('', - '' + xml.push( + '', + '', ); if (versioningConfiguration && versioningConfiguration.Status) { @@ -59,13 +59,10 @@ function bucketGetVersioning(authInfo, request, log, callback) { }; standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => { - const corsHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); if (err) { - log.debug('error processing request', - { method: 'bucketGetVersioning', error: err }); - monitoring.promMetrics( - 'GET', bucketName, err.code, 'getBucketVersioning'); + log.debug('error processing request', { method: 'bucketGetVersioning', error: err }); + monitoring.promMetrics('GET', bucketName, err.code, 'getBucketVersioning'); return callback(err, null, corsHeaders); } const versioningConfiguration = bucket.getVersioningConfiguration(); @@ -74,8 +71,7 @@ function bucketGetVersioning(authInfo, request, log, callback) { authInfo, bucket: bucketName, }); - monitoring.promMetrics( - 'GET', bucketName, '200', 'getBucketVersioning'); + monitoring.promMetrics('GET', bucketName, '200', 'getBucketVersioning'); return callback(null, xml, corsHeaders); }); } diff --git a/lib/api/bucketHead.js b/lib/api/bucketHead.js index bafc52eb2d..506b3f9986 100644 --- a/lib/api/bucketHead.js +++ b/lib/api/bucketHead.js @@ -23,11 +23,9 @@ function bucketHead(authInfo, request, log, callback) { request, }; standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => { - const corsHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); if (err) { - monitoring.promMetrics( - 'HEAD', bucketName, err.code, 'headBucket'); + monitoring.promMetrics('HEAD', bucketName, err.code, 'headBucket'); return callback(err, corsHeaders); } pushMetric('headBucket', log, { diff --git a/lib/api/bucketPutACL.js b/lib/api/bucketPutACL.js index a6e86a4dba..2a2134c6ef 100644 --- a/lib/api/bucketPutACL.js +++ b/lib/api/bucketPutACL.js @@ -54,18 +54,14 @@ function bucketPutACL(authInfo, request, log, callback) { 'authenticated-read', 'log-delivery-write', ]; - const possibleGroups = [constants.allAuthedUsersId, - constants.publicId, - constants.logId, - ]; + const possibleGroups = [constants.allAuthedUsersId, constants.publicId, constants.logId]; const metadataValParams = { authInfo, bucketName, requestType: request.apiMethods || 'bucketPutACL', request, }; - const possibleGrants = ['FULL_CONTROL', 'WRITE', - 'WRITE_ACP', 'READ', 'READ_ACP']; + const possibleGrants = ['FULL_CONTROL', 'WRITE', 'WRITE_ACP', 'READ', 'READ_ACP']; const addACLParams = { Canned: '', FULL_CONTROL: [], @@ -75,235 +71,226 @@ function bucketPutACL(authInfo, request, log, callback) { READ_ACP: [], }; - const grantReadHeader = - aclUtils.parseGrant(request.headers[ - 'x-amz-grant-read'], 'READ'); - const grantWriteHeader = - aclUtils.parseGrant(request.headers['x-amz-grant-write'], 'WRITE'); - const grantReadACPHeader = - aclUtils.parseGrant(request.headers['x-amz-grant-read-acp'], - 'READ_ACP'); - const grantWriteACPHeader = - aclUtils.parseGrant(request.headers['x-amz-grant-write-acp'], - 'WRITE_ACP'); - const grantFullControlHeader = - aclUtils.parseGrant(request.headers['x-amz-grant-full-control'], - 'FULL_CONTROL'); + const grantReadHeader = aclUtils.parseGrant(request.headers['x-amz-grant-read'], 'READ'); + const grantWriteHeader = aclUtils.parseGrant(request.headers['x-amz-grant-write'], 'WRITE'); + const grantReadACPHeader = aclUtils.parseGrant(request.headers['x-amz-grant-read-acp'], 'READ_ACP'); + const grantWriteACPHeader = aclUtils.parseGrant(request.headers['x-amz-grant-write-acp'], 'WRITE_ACP'); + const grantFullControlHeader = aclUtils.parseGrant(request.headers['x-amz-grant-full-control'], 'FULL_CONTROL'); - return async.waterfall([ - function waterfall1(next) { - standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, - (err, bucket) => { - if (err) { - log.trace('request authorization failed', { - error: err, - method: 'metadataValidateBucket', - }); - return next(err, bucket); - } - // if the API call is allowed, ensure that the parameters are valid - if (newCannedACL && possibleCannedACL.indexOf(newCannedACL) === -1) { - log.trace('invalid canned acl argument', { - acl: newCannedACL, - method: 'bucketPutACL', - }); - return next(errors.InvalidArgument); - } - if (!aclUtils.checkGrantHeaderValidity(request.headers)) { - log.trace('invalid acl header'); - return next(errors.InvalidArgument); - } - return next(null, bucket); - }); - }, - function waterfall2(bucket, next) { - // If not setting acl through headers, parse body - if (newCannedACL === undefined - && grantReadHeader === undefined - && grantWriteHeader === undefined - && grantReadACPHeader === undefined - && grantWriteACPHeader === undefined - && grantFullControlHeader === undefined) { - if (request.post) { - log.trace('parsing acls from request body'); - return aclUtils.parseAclXml(request.post, log, - (err, jsonGrants) => next(err, bucket, jsonGrants)); - } - // If no ACLs sent with request at all - return next(errors.MalformedXML, bucket); - } - /** - * If acl set in headers (including canned acl) pass bucket and - * undefined to the next function - */ - log.trace('using acls from request headers'); - return next(null, bucket, undefined); - }, - function waterfall3(bucket, jsonGrants, next) { - // If canned ACL just move on and set them - if (newCannedACL) { - log.trace('canned acl', { cannedAcl: newCannedACL }); - addACLParams.Canned = newCannedACL; - return next(null, bucket, addACLParams); - } - let usersIdentifiedByEmail = []; - let usersIdentifiedByGroup = []; - let usersIdentifiedByID = []; - let hasError = false; - /** - * If grants set by xml, loop through the grants - * and separate grant types so parsed in same manner - * as header grants - */ - if (jsonGrants) { - log.trace('parsing acl grants'); - jsonGrants.forEach(grant => { - const grantee = grant.Grantee[0]; - const granteeType = grantee.$['xsi:type']; - const permission = grant.Permission[0]; - let skip = false; - if (possibleGrants.indexOf(permission) < 0) { - skip = true; - } - if (!skip && granteeType === 'AmazonCustomerByEmail') { - usersIdentifiedByEmail.push({ - identifier: grantee.EmailAddress[0], - grantType: permission, - userIDType: 'emailaddress', + return async.waterfall( + [ + function waterfall1(next) { + standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => { + if (err) { + log.trace('request authorization failed', { + error: err, + method: 'metadataValidateBucket', }); + return next(err, bucket); } - if (!skip && granteeType === 'CanonicalUser') { - usersIdentifiedByID.push({ - identifier: grantee.ID[0], - grantType: permission, - userIDType: 'id', + // if the API call is allowed, ensure that the parameters are valid + if (newCannedACL && possibleCannedACL.indexOf(newCannedACL) === -1) { + log.trace('invalid canned acl argument', { + acl: newCannedACL, + method: 'bucketPutACL', }); + return next(errors.InvalidArgument); } - if (!skip && granteeType === 'Group') { - if (possibleGroups.indexOf(grantee.URI[0]) < 0) { - log.trace('invalid user group', - { userGroup: grantee.URI[0] }); - hasError = true; - return next(errors.InvalidArgument, bucket); - } - return usersIdentifiedByGroup.push({ - identifier: grantee.URI[0], - grantType: permission, - userIDType: 'uri', - }); + if (!aclUtils.checkGrantHeaderValidity(request.headers)) { + log.trace('invalid acl header'); + return next(errors.InvalidArgument); } - return undefined; + return next(null, bucket); }); - if (hasError) { - return undefined; + }, + function waterfall2(bucket, next) { + // If not setting acl through headers, parse body + if ( + newCannedACL === undefined && + grantReadHeader === undefined && + grantWriteHeader === undefined && + grantReadACPHeader === undefined && + grantWriteACPHeader === undefined && + grantFullControlHeader === undefined + ) { + if (request.post) { + log.trace('parsing acls from request body'); + return aclUtils.parseAclXml(request.post, log, (err, jsonGrants) => + next(err, bucket, jsonGrants), + ); + } + // If no ACLs sent with request at all + return next(errors.MalformedXML, bucket); } - } else { - // If no canned ACL and no parsed xml, loop - // through the access headers - const allGrantHeaders = - [].concat(grantReadHeader, grantWriteHeader, - grantReadACPHeader, grantWriteACPHeader, - grantFullControlHeader); + /** + * If acl set in headers (including canned acl) pass bucket and + * undefined to the next function + */ + log.trace('using acls from request headers'); + return next(null, bucket, undefined); + }, + function waterfall3(bucket, jsonGrants, next) { + // If canned ACL just move on and set them + if (newCannedACL) { + log.trace('canned acl', { cannedAcl: newCannedACL }); + addACLParams.Canned = newCannedACL; + return next(null, bucket, addACLParams); + } + let usersIdentifiedByEmail = []; + let usersIdentifiedByGroup = []; + let usersIdentifiedByID = []; + let hasError = false; + /** + * If grants set by xml, loop through the grants + * and separate grant types so parsed in same manner + * as header grants + */ + if (jsonGrants) { + log.trace('parsing acl grants'); + jsonGrants.forEach(grant => { + const grantee = grant.Grantee[0]; + const granteeType = grantee.$['xsi:type']; + const permission = grant.Permission[0]; + let skip = false; + if (possibleGrants.indexOf(permission) < 0) { + skip = true; + } + if (!skip && granteeType === 'AmazonCustomerByEmail') { + usersIdentifiedByEmail.push({ + identifier: grantee.EmailAddress[0], + grantType: permission, + userIDType: 'emailaddress', + }); + } + if (!skip && granteeType === 'CanonicalUser') { + usersIdentifiedByID.push({ + identifier: grantee.ID[0], + grantType: permission, + userIDType: 'id', + }); + } + if (!skip && granteeType === 'Group') { + if (possibleGroups.indexOf(grantee.URI[0]) < 0) { + log.trace('invalid user group', { userGroup: grantee.URI[0] }); + hasError = true; + return next(errors.InvalidArgument, bucket); + } + return usersIdentifiedByGroup.push({ + identifier: grantee.URI[0], + grantType: permission, + userIDType: 'uri', + }); + } + return undefined; + }); + if (hasError) { + return undefined; + } + } else { + // If no canned ACL and no parsed xml, loop + // through the access headers + const allGrantHeaders = [].concat( + grantReadHeader, + grantWriteHeader, + grantReadACPHeader, + grantWriteACPHeader, + grantFullControlHeader, + ); - usersIdentifiedByEmail = allGrantHeaders.filter(item => - item && item.userIDType.toLowerCase() === 'emailaddress'); + usersIdentifiedByEmail = allGrantHeaders.filter( + item => item && item.userIDType.toLowerCase() === 'emailaddress', + ); - usersIdentifiedByGroup = allGrantHeaders - .filter(itm => itm && itm.userIDType - .toLowerCase() === 'uri'); - for (let i = 0; i < usersIdentifiedByGroup.length; i++) { - const userGroup = usersIdentifiedByGroup[i].identifier; - if (possibleGroups.indexOf(userGroup) < 0) { - log.trace('invalid user group', { userGroup, - method: 'bucketPutACL' }); - return next(errors.InvalidArgument, bucket); + usersIdentifiedByGroup = allGrantHeaders.filter( + itm => itm && itm.userIDType.toLowerCase() === 'uri', + ); + for (let i = 0; i < usersIdentifiedByGroup.length; i++) { + const userGroup = usersIdentifiedByGroup[i].identifier; + if (possibleGroups.indexOf(userGroup) < 0) { + log.trace('invalid user group', { userGroup, method: 'bucketPutACL' }); + return next(errors.InvalidArgument, bucket); + } } + /** TODO: Consider whether want to verify with Vault + * whether canonicalID is associated with existing + * account before adding to ACL */ + usersIdentifiedByID = allGrantHeaders.filter( + item => item && item.userIDType.toLowerCase() === 'id', + ); } - /** TODO: Consider whether want to verify with Vault - * whether canonicalID is associated with existing - * account before adding to ACL */ - usersIdentifiedByID = allGrantHeaders - .filter(item => item && item.userIDType - .toLowerCase() === 'id'); - } - // For now, at least make sure ID is 64-char alphanumeric - // string before adding to ACL (check can be removed if - // verifying with Vault for associated accounts first) - for (let i = 0; i < usersIdentifiedByID.length; i++) { - const id = usersIdentifiedByID[i].identifier; - if (!aclUtils.isValidCanonicalId(id)) { - log.trace('invalid user id argument', { - id, - method: 'bucketPutACL', - }); - monitoring.promMetrics('PUT', bucketName, 400, - 'bucketPutACL'); - return callback(errors.InvalidArgument, bucket); + // For now, at least make sure ID is 64-char alphanumeric + // string before adding to ACL (check can be removed if + // verifying with Vault for associated accounts first) + for (let i = 0; i < usersIdentifiedByID.length; i++) { + const id = usersIdentifiedByID[i].identifier; + if (!aclUtils.isValidCanonicalId(id)) { + log.trace('invalid user id argument', { + id, + method: 'bucketPutACL', + }); + monitoring.promMetrics('PUT', bucketName, 400, 'bucketPutACL'); + return callback(errors.InvalidArgument, bucket); + } } - } - const justEmails = usersIdentifiedByEmail - .map(item => item.identifier); - // If have to lookup canonicalID's do that asynchronously - if (justEmails.length > 0) { - return vault.getCanonicalIds(justEmails, log, - (err, results) => { + const justEmails = usersIdentifiedByEmail.map(item => item.identifier); + // If have to lookup canonicalID's do that asynchronously + if (justEmails.length > 0) { + return vault.getCanonicalIds(justEmails, log, (err, results) => { if (err) { log.trace('error looking up canonical ids', { - error: err, method: 'vault.getCanonicalIDs' }); + error: err, + method: 'vault.getCanonicalIDs', + }); return next(err, bucket); } - const reconstructedUsersIdentifiedByEmail = aclUtils - .reconstructUsersIdentifiedByEmail(results, - usersIdentifiedByEmail); + const reconstructedUsersIdentifiedByEmail = aclUtils.reconstructUsersIdentifiedByEmail( + results, + usersIdentifiedByEmail, + ); const allUsers = [].concat( reconstructedUsersIdentifiedByEmail, usersIdentifiedByID, - usersIdentifiedByGroup); - const revisedAddACLParams = aclUtils - .sortHeaderGrants(allUsers, addACLParams); + usersIdentifiedByGroup, + ); + const revisedAddACLParams = aclUtils.sortHeaderGrants(allUsers, addACLParams); return next(null, bucket, revisedAddACLParams); }); + } + const allUsers = [].concat(usersIdentifiedByID, usersIdentifiedByGroup); + const revisedAddACLParams = aclUtils.sortHeaderGrants(allUsers, addACLParams); + return next(null, bucket, revisedAddACLParams); + }, + function waterfall4(bucket, addACLParams, next) { + if (bucket.hasDeletedFlag() && canonicalID !== bucket.getOwner()) { + log.trace('deleted flag on bucket'); + return next(errors.NoSuchBucket); + } + if (bucket.hasTransientFlag() || bucket.hasDeletedFlag()) { + log.trace('transient or deleted flag so cleaning up bucket'); + bucket.setFullAcl(addACLParams); + return cleanUpBucket(bucket, canonicalID, log, err => next(err, bucket)); + } + // If no bucket flags, just add acl's to bucket metadata + return acl.addACL(bucket, addACLParams, log, err => next(err, bucket)); + }, + ], + (err, bucket) => { + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + if (err) { + log.trace('error processing request', { error: err, method: 'bucketPutACL' }); + monitoring.promMetrics('PUT', bucketName, err.code, 'bucketPutACL'); + } else { + pushMetric('putBucketAcl', log, { + authInfo, + bucket: bucketName, + }); + monitoring.promMetrics('PUT', bucketName, '200', 'bucketPutACL'); } - const allUsers = [].concat( - usersIdentifiedByID, - usersIdentifiedByGroup); - const revisedAddACLParams = - aclUtils.sortHeaderGrants(allUsers, addACLParams); - return next(null, bucket, revisedAddACLParams); - }, - function waterfall4(bucket, addACLParams, next) { - if (bucket.hasDeletedFlag() && canonicalID !== bucket.getOwner()) { - log.trace('deleted flag on bucket'); - return next(errors.NoSuchBucket); - } - if (bucket.hasTransientFlag() || bucket.hasDeletedFlag()) { - log.trace('transient or deleted flag so cleaning up bucket'); - bucket.setFullAcl(addACLParams); - return cleanUpBucket(bucket, canonicalID, log, err => - next(err, bucket)); - } - // If no bucket flags, just add acl's to bucket metadata - return acl.addACL(bucket, addACLParams, log, err => - next(err, bucket)); + return callback(err, corsHeaders); }, - ], (err, bucket) => { - const corsHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); - if (err) { - log.trace('error processing request', { error: err, - method: 'bucketPutACL' }); - monitoring.promMetrics('PUT', bucketName, err.code, 'bucketPutACL'); - } else { - pushMetric('putBucketAcl', log, { - authInfo, - bucket: bucketName, - }); - monitoring.promMetrics('PUT', bucketName, '200', 'bucketPutACL'); - } - return callback(err, corsHeaders); - }); + ); } module.exports = bucketPutACL; diff --git a/lib/api/bucketPutCors.js b/lib/api/bucketPutCors.js index 2cc05d25ad..e17cc53332 100644 --- a/lib/api/bucketPutCors.js +++ b/lib/api/bucketPutCors.js @@ -30,8 +30,7 @@ function bucketPutCors(authInfo, request, log, callback) { }; if (!request.post) { - log.debug('CORS xml body is missing', - { error: errors.MissingRequestBodyError }); + log.debug('CORS xml body is missing', { error: errors.MissingRequestBodyError }); monitoring.promMetrics('PUT', bucketName, 400, METRICS_ACTION); return callback(errors.MissingRequestBodyError); } @@ -43,38 +42,41 @@ function bucketPutCors(authInfo, request, log, callback) { return callback(errorInstances.MalformedXML.customizeDescription(errMsg)); } - return async.waterfall([ - next => { - log.trace('parsing cors rules'); - return parseCorsXml(request.post, log, next); - }, - (rules, next) => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, - (err, bucket) => { - const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); - if (err) { - monitoring.promMetrics('PUT', bucketName, err.code, METRICS_ACTION); - if (err?.is?.AccessDenied) { - return next(err, corsHeaders); + return async.waterfall( + [ + next => { + log.trace('parsing cors rules'); + return parseCorsXml(request.post, log, next); + }, + (rules, next) => + standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => { + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + if (err) { + monitoring.promMetrics('PUT', bucketName, err.code, METRICS_ACTION); + if (err?.is?.AccessDenied) { + return next(err, corsHeaders); + } + return next(err); } - return next(err); - } - return next(null, bucket, rules, corsHeaders); - }), - (bucket, rules, corsHeaders, next) => { - bucket.setCors(rules); - return metadata.updateBucket(bucketName, bucket, log, err => next(err, corsHeaders)); + return next(null, bucket, rules, corsHeaders); + }), + (bucket, rules, corsHeaders, next) => { + bucket.setCors(rules); + return metadata.updateBucket(bucketName, bucket, log, err => next(err, corsHeaders)); + }, + ], + (err, corsHeaders) => { + if (err) { + log.trace('error processing request', { error: err, method: 'bucketPutCors' }); + monitoring.promMetrics('PUT', bucketName, err.code, METRICS_ACTION); + return callback(err, corsHeaders); + } + pushMetric(METRICS_ACTION, log, { authInfo, bucket: bucketName }); + monitoring.promMetrics('PUT', bucketName, '200', METRICS_ACTION); + return callback(null, corsHeaders); }, - ], (err, corsHeaders) => { - if (err) { - log.trace('error processing request', { error: err, method: 'bucketPutCors' }); - monitoring.promMetrics('PUT', bucketName, err.code, METRICS_ACTION); - return callback(err, corsHeaders); - } - pushMetric(METRICS_ACTION, log, { authInfo, bucket: bucketName }); - monitoring.promMetrics('PUT', bucketName, '200', METRICS_ACTION); - return callback(null, corsHeaders); - }); + ); } module.exports = bucketPutCors; diff --git a/lib/api/bucketPutEncryption.js b/lib/api/bucketPutEncryption.js index e214407ed7..786fa5ad72 100644 --- a/lib/api/bucketPutEncryption.js +++ b/lib/api/bucketPutEncryption.js @@ -27,73 +27,74 @@ function bucketPutEncryption(authInfo, request, log, callback) { request, }; - return async.waterfall([ - next => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, next), - (bucket, next) => checkExpectedBucketOwner(request.headers, bucket, log, err => next(err, bucket)), - (bucket, next) => { - log.trace('parsing encryption config', { method: 'bucketPutEncryption' }); - return parseEncryptionXml(request.post, log, (err, encryptionConfig) => { - if (err) { - return next(err); - } - return next(null, bucket, encryptionConfig); - }); - }, - (bucket, encryptionConfig, next) => { - const existingConfig = bucket.getServerSideEncryption(); - // Check if encryption is not configured or if a default master key has not been created yet. - if (existingConfig === null || !existingConfig.masterKeyId) { - return kms.bucketLevelEncryption(bucket, encryptionConfig, log, - (err, updatedConfig) => { + return async.waterfall( + [ + next => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, next), + (bucket, next) => checkExpectedBucketOwner(request.headers, bucket, log, err => next(err, bucket)), + (bucket, next) => { + log.trace('parsing encryption config', { method: 'bucketPutEncryption' }); + return parseEncryptionXml(request.post, log, (err, encryptionConfig) => { + if (err) { + return next(err); + } + return next(null, bucket, encryptionConfig); + }); + }, + (bucket, encryptionConfig, next) => { + const existingConfig = bucket.getServerSideEncryption(); + // Check if encryption is not configured or if a default master key has not been created yet. + if (existingConfig === null || !existingConfig.masterKeyId) { + return kms.bucketLevelEncryption(bucket, encryptionConfig, log, (err, updatedConfig) => { if (err) { return next(err); } return next(null, bucket, updatedConfig); }); - } + } - // If encryption is already configured and a default master key exists + // If encryption is already configured and a default master key exists - // If the request does not specify a custom key, reuse the existing default master key id - // This ensures that a new default master key is not generated every time - // `putBucketEncryption` is called, avoiding unnecessary key creation - const updatedConfig = { - mandatory: true, - algorithm: encryptionConfig.algorithm, - cryptoScheme: existingConfig.cryptoScheme, - masterKeyId: existingConfig.masterKeyId, - }; + // If the request does not specify a custom key, reuse the existing default master key id + // This ensures that a new default master key is not generated every time + // `putBucketEncryption` is called, avoiding unnecessary key creation + const updatedConfig = { + mandatory: true, + algorithm: encryptionConfig.algorithm, + cryptoScheme: existingConfig.cryptoScheme, + masterKeyId: existingConfig.masterKeyId, + }; - // If the request specifies a custom master key id, store it in the updated configuration - const { configuredMasterKeyId } = encryptionConfig; - if (configuredMasterKeyId) { - updatedConfig.configuredMasterKeyId = configuredMasterKeyId; - } + // If the request specifies a custom master key id, store it in the updated configuration + const { configuredMasterKeyId } = encryptionConfig; + if (configuredMasterKeyId) { + updatedConfig.configuredMasterKeyId = configuredMasterKeyId; + } - const { isAccountEncryptionEnabled } = existingConfig; - if (isAccountEncryptionEnabled) { - updatedConfig.isAccountEncryptionEnabled = isAccountEncryptionEnabled; - } + const { isAccountEncryptionEnabled } = existingConfig; + if (isAccountEncryptionEnabled) { + updatedConfig.isAccountEncryptionEnabled = isAccountEncryptionEnabled; + } - return next(null, bucket, updatedConfig); - }, - (bucket, updatedConfig, next) => { - bucket.setServerSideEncryption(updatedConfig); - metadata.updateBucket(bucket.getName(), bucket, log, err => next(err, bucket)); + return next(null, bucket, updatedConfig); + }, + (bucket, updatedConfig, next) => { + bucket.setServerSideEncryption(updatedConfig); + metadata.updateBucket(bucket.getName(), bucket, log, err => next(err, bucket)); + }, + ], + (err, bucket) => { + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + if (err) { + log.trace('error processing request', { error: err, method: 'bucketPutEncryption' }); + return callback(err, corsHeaders); + } + pushMetric('putBucketEncryption', log, { + authInfo, + bucket: bucketName, + }); + return callback(null, corsHeaders); }, - ], - (err, bucket) => { - const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); - if (err) { - log.trace('error processing request', { error: err, method: 'bucketPutEncryption' }); - return callback(err, corsHeaders); - } - pushMetric('putBucketEncryption', log, { - authInfo, - bucket: bucketName, - }); - return callback(null, corsHeaders); - }); + ); } module.exports = bucketPutEncryption; diff --git a/lib/api/bucketPutLifecycle.js b/lib/api/bucketPutLifecycle.js index b1a30e27bf..9e0ad00aad 100644 --- a/lib/api/bucketPutLifecycle.js +++ b/lib/api/bucketPutLifecycle.js @@ -1,7 +1,6 @@ const { waterfall } = require('async'); const uuid = require('uuid').v4; -const LifecycleConfiguration = - require('arsenal').models.LifecycleConfiguration; +const LifecycleConfiguration = require('arsenal').models.LifecycleConfiguration; const config = require('../Config').config; const parseXML = require('../utilities/parseXML'); @@ -30,53 +29,51 @@ function bucketPutLifecycle(authInfo, request, log, callback) { requestType: request.apiMethods || 'bucketPutLifecycle', request, }; - return waterfall([ - next => parseXML(request.post, log, next), - (parsedXml, next) => { - const lcConfigClass = - new LifecycleConfiguration(parsedXml, config); - // if there was an error getting lifecycle configuration, - // returned configObj will contain 'error' key - process.nextTick(() => { - const configObj = lcConfigClass.getLifecycleConfiguration(); - if (configObj.error) { - return next(configObj.error); + return waterfall( + [ + next => parseXML(request.post, log, next), + (parsedXml, next) => { + const lcConfigClass = new LifecycleConfiguration(parsedXml, config); + // if there was an error getting lifecycle configuration, + // returned configObj will contain 'error' key + process.nextTick(() => { + const configObj = lcConfigClass.getLifecycleConfiguration(); + if (configObj.error) { + return next(configObj.error); + } + return next(null, configObj); + }); + }, + (lcConfig, next) => + standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => { + if (err) { + return next(err, bucket); + } + return next(null, bucket, lcConfig); + }), + (bucket, lcConfig, next) => { + if (!bucket.getUid()) { + bucket.setUid(uuid()); } - return next(null, configObj); - }); - }, - (lcConfig, next) => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, - (err, bucket) => { - if (err) { - return next(err, bucket); - } - return next(null, bucket, lcConfig); - }), - (bucket, lcConfig, next) => { - if (!bucket.getUid()) { - bucket.setUid(uuid()); + bucket.setLifecycleConfiguration(lcConfig); + metadata.updateBucket(bucket.getName(), bucket, log, err => next(err, bucket)); + }, + ], + (err, bucket) => { + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + if (err) { + log.trace('error processing request', { error: err, method: 'bucketPutLifecycle' }); + monitoring.promMetrics('PUT', bucketName, err.code, 'putBucketLifecycle'); + return callback(err, corsHeaders); } - bucket.setLifecycleConfiguration(lcConfig); - metadata.updateBucket(bucket.getName(), bucket, log, err => - next(err, bucket)); + pushMetric('putBucketLifecycle', log, { + authInfo, + bucket: bucketName, + }); + monitoring.promMetrics('PUT', bucketName, '200', 'putBucketLifecycle'); + return callback(null, corsHeaders); }, - ], (err, bucket) => { - const corsHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); - if (err) { - log.trace('error processing request', { error: err, - method: 'bucketPutLifecycle' }); - monitoring.promMetrics( - 'PUT', bucketName, err.code, 'putBucketLifecycle'); - return callback(err, corsHeaders); - } - pushMetric('putBucketLifecycle', log, { - authInfo, - bucket: bucketName, - }); - monitoring.promMetrics('PUT', bucketName, '200', 'putBucketLifecycle'); - return callback(null, corsHeaders); - }); + ); } module.exports = bucketPutLifecycle; diff --git a/lib/api/bucketPutLogging.js b/lib/api/bucketPutLogging.js index 3592f4432a..0c51dfeb52 100644 --- a/lib/api/bucketPutLogging.js +++ b/lib/api/bucketPutLogging.js @@ -7,7 +7,7 @@ const monitoring = require('../utilities/monitoringHandler'); const { errorInstances } = require('arsenal'); const { config, serverAccessLogsModes } = require('../Config'); -const ERROR_MSG_SOURCE_TARGET_BUCKET_OWNER_MISMATCH = +const ERROR_MSG_SOURCE_TARGET_BUCKET_OWNER_MISMATCH = 'The owner for the bucket to be logged and the target bucket must be the same.'; function bucketPutLogging(authInfo, request, log, callback) { @@ -32,60 +32,68 @@ function bucketPutLogging(authInfo, request, log, callback) { request, }; - return waterfall([ - next => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => { - if (err) { - return next(err); - } - - return next(null, bucket); - }), - (bucket, next) => { - const loggingEnabled = parsed.res.getLoggingEnabled(); - if (!loggingEnabled) { - return next(null, bucket); - } + return waterfall( + [ + next => + standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => { + if (err) { + return next(err); + } - return metadata.getBucket(loggingEnabled.TargetBucket, log, (err, targetBucket) => { - if (err) { - return next(errorInstances.InvalidTargetBucketForLogging.customizeDescription(err.description)); + return next(null, bucket); + }), + (bucket, next) => { + const loggingEnabled = parsed.res.getLoggingEnabled(); + if (!loggingEnabled) { + return next(null, bucket); } - if (targetBucket.getOwner() !== bucket.getOwner()) { - return next(errorInstances.InvalidTargetBucketForLogging - .customizeDescription(ERROR_MSG_SOURCE_TARGET_BUCKET_OWNER_MISMATCH)); - } + return metadata.getBucket(loggingEnabled.TargetBucket, log, (err, targetBucket) => { + if (err) { + return next(errorInstances.InvalidTargetBucketForLogging.customizeDescription(err.description)); + } - return next(null, bucket); - }); - }, - (bucket, next) => checkExpectedBucketOwner(request.headers, bucket, log, err => { + if (targetBucket.getOwner() !== bucket.getOwner()) { + return next( + errorInstances.InvalidTargetBucketForLogging.customizeDescription( + ERROR_MSG_SOURCE_TARGET_BUCKET_OWNER_MISMATCH, + ), + ); + } + + return next(null, bucket); + }); + }, + (bucket, next) => + checkExpectedBucketOwner(request.headers, bucket, log, err => { + if (err) { + return next(err); + } + + return next(null, bucket); + }), + (bucket, next) => { + bucket.setBucketLoggingStatus(parsed.res); + return metadata.updateBucket(bucket.getName(), bucket, log, err => { + if (err) { + return next(err); + } + + return next(); + }); + }, + ], + err => { if (err) { - return next(err); + log.trace('error processing request', { error: err, method: 'bucketPutLogging' }); + monitoring.promMetrics('PUT', bucketName, err.code, 'bucketPutLogging'); + return callback(err); } - return next(null, bucket); - }), - (bucket, next) => { - bucket.setBucketLoggingStatus(parsed.res); - return metadata.updateBucket(bucket.getName(), bucket, log, err => { - if (err) { - return next(err); - } - - return next(); - }); + monitoring.promMetrics('PUT', bucketName, '200', 'bucketPutLogging'); + return callback(); }, - ], err => { - if (err) { - log.trace('error processing request', { error: err, method: 'bucketPutLogging' }); - monitoring.promMetrics('PUT', bucketName, err.code, 'bucketPutLogging'); - return callback(err); - } - - monitoring.promMetrics('PUT', bucketName, '200', 'bucketPutLogging'); - return callback(); - }); + ); } module.exports = bucketPutLogging; diff --git a/lib/api/bucketPutNotification.js b/lib/api/bucketPutNotification.js index ab82b5109a..4719f88fab 100644 --- a/lib/api/bucketPutNotification.js +++ b/lib/api/bucketPutNotification.js @@ -27,34 +27,36 @@ function bucketPutNotification(authInfo, request, log, callback) { request, }; - return async.waterfall([ - next => parseXML(request.post, log, next), - (parsedXml, next) => { - const notificationConfig = getNotificationConfiguration(parsedXml); - const notifConfig = notificationConfig.error ? undefined : notificationConfig; - process.nextTick(() => next(notificationConfig.error, notifConfig)); + return async.waterfall( + [ + next => parseXML(request.post, log, next), + (parsedXml, next) => { + const notificationConfig = getNotificationConfiguration(parsedXml); + const notifConfig = notificationConfig.error ? undefined : notificationConfig; + process.nextTick(() => next(notificationConfig.error, notifConfig)); + }, + (notifConfig, next) => + standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => + next(err, bucket, notifConfig), + ), + (bucket, notifConfig, next) => { + bucket.setNotificationConfiguration(notifConfig); + metadata.updateBucket(bucket.getName(), bucket, log, err => next(err, bucket)); + }, + ], + (err, bucket) => { + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + if (err) { + log.trace('error processing request', { error: err, method: 'bucketPutNotification' }); + return callback(err, corsHeaders); + } + pushMetric('putBucketNotification', log, { + authInfo, + bucket: bucketName, + }); + return callback(null, corsHeaders); }, - (notifConfig, next) => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, - (err, bucket) => next(err, bucket, notifConfig)), - (bucket, notifConfig, next) => { - bucket.setNotificationConfiguration(notifConfig); - metadata.updateBucket(bucket.getName(), bucket, log, - err => next(err, bucket)); - }, - ], (err, bucket) => { - const corsHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); - if (err) { - log.trace('error processing request', { error: err, - method: 'bucketPutNotification' }); - return callback(err, corsHeaders); - } - pushMetric('putBucketNotification', log, { - authInfo, - bucket: bucketName, - }); - return callback(null, corsHeaders); - }); + ); } module.exports = bucketPutNotification; diff --git a/lib/api/bucketPutObjectLock.js b/lib/api/bucketPutObjectLock.js index a3d549789a..cb7b20bd63 100644 --- a/lib/api/bucketPutObjectLock.js +++ b/lib/api/bucketPutObjectLock.js @@ -2,7 +2,9 @@ const { waterfall } = require('async'); const arsenal = require('arsenal'); const { errorInstances } = arsenal; -const { models: { ObjectLockConfiguration } } = arsenal; +const { + models: { ObjectLockConfiguration }, +} = arsenal; const parseXML = require('../utilities/parseXML'); const collectCorsHeaders = require('../utilities/collectCorsHeaders'); @@ -29,55 +31,57 @@ function bucketPutObjectLock(authInfo, request, log, callback) { requestType: request.apiMethods || 'bucketPutObjectLock', request, }; - return waterfall([ - next => parseXML(request.post, log, next), - (parsedXml, next) => { - const lockConfigClass = new ObjectLockConfiguration(parsedXml); - // if there was an error getting object lock configuration, - // returned configObj will contain 'error' key - process.nextTick(() => { - const configObj = lockConfigClass. - getValidatedObjectLockConfiguration(); - return next(configObj.error || null, configObj); + return waterfall( + [ + next => parseXML(request.post, log, next), + (parsedXml, next) => { + const lockConfigClass = new ObjectLockConfiguration(parsedXml); + // if there was an error getting object lock configuration, + // returned configObj will contain 'error' key + process.nextTick(() => { + const configObj = lockConfigClass.getValidatedObjectLockConfiguration(); + return next(configObj.error || null, configObj); + }); + }, + (objectLockConfig, next) => + standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => { + if (err) { + return next(err, bucket); + } + return next(null, bucket, objectLockConfig); + }), + (bucket, objectLockConfig, next) => { + const isObjectLockEnabled = bucket.isObjectLockEnabled(); + process.nextTick(() => { + if (!isObjectLockEnabled) { + return next( + errorInstances.InvalidBucketState.customizeDescription( + 'Object Lock configuration cannot be enabled on ' + 'existing buckets', + ), + bucket, + ); + } + return next(null, bucket, objectLockConfig); + }); + }, + (bucket, objectLockConfig, next) => { + bucket.setObjectLockConfiguration(objectLockConfig); + metadata.updateBucket(bucket.getName(), bucket, log, err => next(err, bucket)); + }, + ], + (err, bucket) => { + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + if (err) { + log.trace('error processing request', { error: err, method: 'bucketPutObjectLock' }); + return callback(err, corsHeaders); + } + pushMetric('putBucketObjectLock', log, { + authInfo, + bucket: bucketName, }); + return callback(null, corsHeaders); }, - (objectLockConfig, next) => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, - log, (err, bucket) => { - if (err) { - return next(err, bucket); - } - return next(null, bucket, objectLockConfig); - }), - (bucket, objectLockConfig, next) => { - const isObjectLockEnabled = bucket.isObjectLockEnabled(); - process.nextTick(() => { - if (!isObjectLockEnabled) { - return next(errorInstances.InvalidBucketState.customizeDescription( - 'Object Lock configuration cannot be enabled on ' + - 'existing buckets'), bucket); - } - return next(null, bucket, objectLockConfig); - }); - }, - (bucket, objectLockConfig, next) => { - bucket.setObjectLockConfiguration(objectLockConfig); - metadata.updateBucket(bucket.getName(), bucket, log, err => - next(err, bucket)); - }, - ], (err, bucket) => { - const corsHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); - if (err) { - log.trace('error processing request', { error: err, - method: 'bucketPutObjectLock' }); - return callback(err, corsHeaders); - } - pushMetric('putBucketObjectLock', log, { - authInfo, - bucket: bucketName, - }); - return callback(null, corsHeaders); - }); + ); } module.exports = bucketPutObjectLock; diff --git a/lib/api/bucketPutPolicy.js b/lib/api/bucketPutPolicy.js index 56f48e2a7f..157171303b 100644 --- a/lib/api/bucketPutPolicy.js +++ b/lib/api/bucketPutPolicy.js @@ -3,8 +3,7 @@ const { errorInstances, models } = require('arsenal'); const collectCorsHeaders = require('../utilities/collectCorsHeaders'); const metadata = require('../metadata/wrapper'); const { standardMetadataValidateBucket } = require('../metadata/metadataUtils'); -const { validatePolicyResource, validatePolicyConditions } = - require('./apiUtils/authorization/permissionChecks'); +const { validatePolicyResource, validatePolicyConditions } = require('./apiUtils/authorization/permissionChecks'); const { BucketPolicy } = models; /** @@ -16,8 +15,7 @@ const { BucketPolicy } = models; function _checkNotImplementedPolicy(policyString) { // bucket names and key names cannot include "", so including those // isolates not implemented keys - return policyString.includes('"Service"') - || policyString.includes('"Federated"'); + return policyString.includes('"Service"') || policyString.includes('"Federated"'); } /** @@ -39,58 +37,57 @@ function bucketPutPolicy(authInfo, request, log, callback) { request, }; - return async.waterfall([ - next => { - const bucketPolicy = new BucketPolicy(request.post); - // if there was an error getting bucket policy, - // returned policyObj will contain 'error' key - process.nextTick(() => { - const policyObj = bucketPolicy.getBucketPolicy(); - if (_checkNotImplementedPolicy(request.post)) { - const err = errorInstances.NotImplemented.customizeDescription( - 'Bucket policy contains element not yet implemented'); - return next(err); - } - if (policyObj.error) { - const err = errorInstances.MalformedPolicy.customizeDescription( - policyObj.error.description); - return next(err); - } - return next(null, policyObj); - }); + return async.waterfall( + [ + next => { + const bucketPolicy = new BucketPolicy(request.post); + // if there was an error getting bucket policy, + // returned policyObj will contain 'error' key + process.nextTick(() => { + const policyObj = bucketPolicy.getBucketPolicy(); + if (_checkNotImplementedPolicy(request.post)) { + const err = errorInstances.NotImplemented.customizeDescription( + 'Bucket policy contains element not yet implemented', + ); + return next(err); + } + if (policyObj.error) { + const err = errorInstances.MalformedPolicy.customizeDescription(policyObj.error.description); + return next(err); + } + return next(null, policyObj); + }); + }, + (bucketPolicy, next) => { + process.nextTick(() => { + if (!validatePolicyResource(bucketName, bucketPolicy)) { + return next(errorInstances.MalformedPolicy.customizeDescription('Policy has invalid resource')); + } + return next(validatePolicyConditions(bucketPolicy), bucketPolicy); + }); + }, + (bucketPolicy, next) => + standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => { + if (err) { + return next(err, bucket); + } + return next(null, bucket, bucketPolicy); + }), + (bucket, bucketPolicy, next) => { + bucket.setBucketPolicy(bucketPolicy); + metadata.updateBucket(bucket.getName(), bucket, log, err => next(err, bucket)); + }, + ], + (err, bucket) => { + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + if (err) { + log.trace('error processing request', { error: err, method: 'bucketPutPolicy' }); + return callback(err, corsHeaders); + } + // TODO: implement Utapi metric support + return callback(null, corsHeaders); }, - (bucketPolicy, next) => { - process.nextTick(() => { - if (!validatePolicyResource(bucketName, bucketPolicy)) { - return next(errorInstances.MalformedPolicy.customizeDescription( - 'Policy has invalid resource')); - } - return next(validatePolicyConditions(bucketPolicy), bucketPolicy); - }); - }, - (bucketPolicy, next) => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, - (err, bucket) => { - if (err) { - return next(err, bucket); - } - return next(null, bucket, bucketPolicy); - }), - (bucket, bucketPolicy, next) => { - bucket.setBucketPolicy(bucketPolicy); - metadata.updateBucket(bucket.getName(), bucket, log, - err => next(err, bucket)); - }, - ], (err, bucket) => { - const corsHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); - if (err) { - log.trace('error processing request', - { error: err, method: 'bucketPutPolicy' }); - return callback(err, corsHeaders); - } - // TODO: implement Utapi metric support - return callback(null, corsHeaders); - }); + ); } module.exports = bucketPutPolicy; diff --git a/lib/api/bucketPutRateLimit.js b/lib/api/bucketPutRateLimit.js index d895678775..e10b06c2e1 100644 --- a/lib/api/bucketPutRateLimit.js +++ b/lib/api/bucketPutRateLimit.js @@ -20,8 +20,9 @@ function parseRequestBody(requestBody, callback) { function validateRateLimitConfig(requestConfig, callback) { const limit = requestConfig.RequestsPerSecond; if (Number.isNaN(limit) || !Number.isInteger(limit) || limit < 0) { - return callback(errorInstances.InvalidArgument - .customizeDescription('RequestsPerSecond must be a positive integer')); + return callback( + errorInstances.InvalidArgument.customizeDescription('RequestsPerSecond must be a positive integer'), + ); } // Validate limit against node count AND worker count @@ -30,21 +31,25 @@ function validateRateLimitConfig(requestConfig, callback) { const minLimit = nodes * workers; if (limit > 0 && limit < minLimit) { - return callback(errorInstances.InvalidArgument - .customizeDescription( + return callback( + errorInstances.InvalidArgument.customizeDescription( `RequestsPerSecond (${limit}) must be >= ` + - `(nodes x workers = ${nodes} x ${workers} = ${minLimit}) or 0 (unlimited). ` + - 'Each worker enforces limit/nodes/workers locally. ' + - `With limit less than ${minLimit}, per-worker rate would be less than 1 req/s, ` + - 'effectively blocking traffic.' - )); + `(nodes x workers = ${nodes} x ${workers} = ${minLimit}) or 0 (unlimited). ` + + 'Each worker enforces limit/nodes/workers locally. ' + + `With limit less than ${minLimit}, per-worker rate would be less than 1 req/s, ` + + 'effectively blocking traffic.', + ), + ); } - return callback(null, new models.RateLimitConfiguration({ - RequestsPerSecond: { - Limit: limit, - }, - })); + return callback( + null, + new models.RateLimitConfiguration({ + RequestsPerSecond: { + Limit: limit, + }, + }), + ); } /** @@ -70,35 +75,35 @@ function bucketPutRateLimit(authInfo, request, log, callback) { request, }; - return async.waterfall([ - next => parseRequestBody(request.post, next), - (requestBody, next) => validateRateLimitConfig(requestBody, next), - (limitConfig, next) => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, - (err, bucket) => { - if (err) { - return next(err, bucket); - } - return next(null, bucket, limitConfig); - }), - (bucket, limitConfig, next) => { - bucket.setRateLimitConfiguration(limitConfig); - metadata.updateBucket(bucket.getName(), bucket, log, - err => next(err, bucket)); + return async.waterfall( + [ + next => parseRequestBody(request.post, next), + (requestBody, next) => validateRateLimitConfig(requestBody, next), + (limitConfig, next) => + standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => { + if (err) { + return next(err, bucket); + } + return next(null, bucket, limitConfig); + }), + (bucket, limitConfig, next) => { + bucket.setRateLimitConfiguration(limitConfig); + metadata.updateBucket(bucket.getName(), bucket, log, err => next(err, bucket)); + }, + ], + (err, bucket) => { + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + if (err) { + log.trace('error processing request', { error: err, method: 'bucketPutRateLimit' }); + return callback(err, corsHeaders); + } + // Invalidate cache so new limit takes effect immediately + cache.deleteCachedConfig(cache.namespace.bucket, bucketName); + log.debug('invalidated rate limit cache for bucket', { bucketName }); + // TODO: implement Utapi metric support + return callback(null, corsHeaders); }, - ], (err, bucket) => { - const corsHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); - if (err) { - log.trace('error processing request', - { error: err, method: 'bucketPutRateLimit' }); - return callback(err, corsHeaders); - } - // Invalidate cache so new limit takes effect immediately - cache.deleteCachedConfig(cache.namespace.bucket, bucketName); - log.debug('invalidated rate limit cache for bucket', { bucketName }); - // TODO: implement Utapi metric support - return callback(null, corsHeaders); - }); + ); } module.exports = bucketPutRateLimit; diff --git a/lib/api/bucketPutReplication.js b/lib/api/bucketPutReplication.js index c940e660b4..a5a62794a0 100644 --- a/lib/api/bucketPutReplication.js +++ b/lib/api/bucketPutReplication.js @@ -4,17 +4,15 @@ const { errorInstances } = require('arsenal'); const metadata = require('../metadata/wrapper'); const { standardMetadataValidateBucket } = require('../metadata/metadataUtils'); const { pushMetric } = require('../utapi/utilities'); -const { getReplicationConfiguration } = - require('./apiUtils/bucket/getReplicationConfiguration'); -const validateConfiguration = - require('./apiUtils/bucket/validateReplicationConfig'); +const { getReplicationConfiguration } = require('./apiUtils/bucket/getReplicationConfiguration'); +const validateConfiguration = require('./apiUtils/bucket/validateReplicationConfig'); const collectCorsHeaders = require('../utilities/collectCorsHeaders'); const monitoring = require('../utilities/monitoringHandler'); // The error response when a bucket does not have versioning 'Enabled'. const versioningNotEnabledError = errorInstances.InvalidRequest.customizeDescription( - 'Versioning must be \'Enabled\' on the bucket to apply a replication ' + - 'configuration'); + "Versioning must be 'Enabled' on the bucket to apply a replication " + 'configuration', +); /** * bucketPutReplication - Create or update bucket replication configuration @@ -34,57 +32,55 @@ function bucketPutReplication(authInfo, request, log, callback) { request, }; - return waterfall([ - // Validate the request XML and return the replication configuration. - next => getReplicationConfiguration(post, log, next), - // Check bucket user privileges and ensure versioning is 'Enabled'. - (config, next) => - // TODO: Validate that destination bucket exists and has versioning. - standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => { - if (err) { - return next(err); + return waterfall( + [ + // Validate the request XML and return the replication configuration. + next => getReplicationConfiguration(post, log, next), + // Check bucket user privileges and ensure versioning is 'Enabled'. + (config, next) => + // TODO: Validate that destination bucket exists and has versioning. + standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => { + if (err) { + return next(err); + } + // Replication requires that versioning is 'Enabled' unless it + // is an NFS bucket. + if (!bucket.isNFS() && !bucket.isVersioningEnabled(bucket)) { + return next(versioningNotEnabledError); + } + return next(null, config, bucket); + }), + // Set the replication configuration and update the bucket metadata. + (config, bucket, next) => { + // validate there's a preferred read location in case the + // bucket location is a transient source + if (!validateConfiguration(config, bucket)) { + const msg = 'Replication configuration lacks a preferred ' + 'read location'; + log.error(msg, { bucketName: bucket.getName() }); + return next(errorInstances.ValidationError.customizeDescription(msg)); } - // Replication requires that versioning is 'Enabled' unless it - // is an NFS bucket. - if (!bucket.isNFS() && !bucket.isVersioningEnabled(bucket)) { - return next(versioningNotEnabledError); - } - return next(null, config, bucket); - }), - // Set the replication configuration and update the bucket metadata. - (config, bucket, next) => { - // validate there's a preferred read location in case the - // bucket location is a transient source - if (!validateConfiguration(config, bucket)) { - const msg = 'Replication configuration lacks a preferred ' + - 'read location'; - log.error(msg, { bucketName: bucket.getName() }); - return next(errorInstances.ValidationError - .customizeDescription(msg)); + bucket.setReplicationConfiguration(config); + return metadata.updateBucket(bucket.getName(), bucket, log, err => next(err, bucket)); + }, + ], + (err, bucket) => { + const corsHeaders = collectCorsHeaders(headers.origin, method, bucket); + if (err) { + log.trace('error processing request', { + error: err, + method: 'bucketPutReplication', + }); + monitoring.promMetrics('PUT', bucketName, err.code, 'putBucketReplication'); + return callback(err, corsHeaders); } - bucket.setReplicationConfiguration(config); - return metadata.updateBucket(bucket.getName(), bucket, log, err => - next(err, bucket)); - }, - ], (err, bucket) => { - const corsHeaders = collectCorsHeaders(headers.origin, method, bucket); - if (err) { - log.trace('error processing request', { - error: err, - method: 'bucketPutReplication', + pushMetric('putBucketReplication', log, { + authInfo, + bucket: bucketName, }); - monitoring.promMetrics( - 'PUT', bucketName, err.code, 'putBucketReplication'); - return callback(err, corsHeaders); - } - pushMetric('putBucketReplication', log, { - authInfo, - bucket: bucketName, - }); - monitoring.promMetrics( - 'PUT', bucketName, '200', 'putBucketReplication'); - return callback(null, corsHeaders); - }); + monitoring.promMetrics('PUT', bucketName, '200', 'putBucketReplication'); + return callback(null, corsHeaders); + }, + ); } module.exports = bucketPutReplication; diff --git a/lib/api/bucketPutTagging.js b/lib/api/bucketPutTagging.js index 9023f48504..292133d223 100644 --- a/lib/api/bucketPutTagging.js +++ b/lib/api/bucketPutTagging.js @@ -1,7 +1,6 @@ const { waterfall } = require('async'); const { s3middleware } = require('arsenal'); - const collectCorsHeaders = require('../utilities/collectCorsHeaders'); const { standardMetadataValidateBucket } = require('../metadata/metadataUtils'); const metadata = require('../metadata/wrapper'); @@ -42,43 +41,42 @@ function bucketPutTagging(authInfo, request, log, callback) { request, }; let bucket = null; - return waterfall([ - next => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, - (err, b) => { - bucket = b; - return next(err); - }), - next => checkExpectedBucketOwner(headers, bucket, log, next), - next => parseTagXml(request.post, log, next), - (tags, next) => { - const tagArray = []; - Object.keys(tags).forEach(key => { - tagArray.push({ Value: tags[key], Key: key }); - }); - bucket.setTags(tagArray); - metadata.updateBucket(bucket.getName(), bucket, log, err => - next(err)); + return waterfall( + [ + next => + standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, b) => { + bucket = b; + return next(err); + }), + next => checkExpectedBucketOwner(headers, bucket, log, next), + next => parseTagXml(request.post, log, next), + (tags, next) => { + const tagArray = []; + Object.keys(tags).forEach(key => { + tagArray.push({ Value: tags[key], Key: key }); + }); + bucket.setTags(tagArray); + metadata.updateBucket(bucket.getName(), bucket, log, err => next(err)); + }, + ], + err => { + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + if (err) { + log.debug('error processing request', { + error: err, + method: 'bucketPutTagging', + }); + monitoring.promMetrics('PUT', bucketName, err.code, 'putBucketTagging'); + } else { + monitoring.promMetrics('PUT', bucketName, '200', 'putBucketTagging'); + pushMetric('putBucketTagging', log, { + authInfo, + bucket: bucketName, + }); + } + return callback(err, corsHeaders); }, - ], err => { - const corsHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); - if (err) { - log.debug('error processing request', { - error: err, - method: 'bucketPutTagging' - }); - monitoring.promMetrics('PUT', bucketName, err.code, - 'putBucketTagging'); - } else { - monitoring.promMetrics( - 'PUT', bucketName, '200', 'putBucketTagging'); - pushMetric('putBucketTagging', log, { - authInfo, - bucket: bucketName, - }); - } - return callback(err, corsHeaders); - }); + ); } module.exports = bucketPutTagging; diff --git a/lib/api/bucketPutVersioning.js b/lib/api/bucketPutVersioning.js index 5f872cc0dd..cb5c6dd07b 100644 --- a/lib/api/bucketPutVersioning.js +++ b/lib/api/bucketPutVersioning.js @@ -6,23 +6,23 @@ const collectCorsHeaders = require('../utilities/collectCorsHeaders'); const metadata = require('../metadata/wrapper'); const { standardMetadataValidateBucket } = require('../metadata/metadataUtils'); const { pushMetric } = require('../utapi/utilities'); -const versioningNotImplBackends = - require('../../constants').versioningNotImplBackends; +const versioningNotImplBackends = require('../../constants').versioningNotImplBackends; const { config } = require('../Config'); const monitoring = require('../utilities/monitoringHandler'); -const externalVersioningErrorMessage = 'We do not currently support putting ' + -'a versioned object to a location-constraint of type Azure or GCP.'; +const externalVersioningErrorMessage = + 'We do not currently support putting ' + 'a versioned object to a location-constraint of type Azure or GCP.'; -const replicationVersioningErrorMessage = 'A replication configuration is ' + -'present on this bucket, so you cannot change the versioning state. To ' + -'change the versioning state, first delete the replication configuration.'; +const replicationVersioningErrorMessage = + 'A replication configuration is ' + + 'present on this bucket, so you cannot change the versioning state. To ' + + 'change the versioning state, first delete the replication configuration.'; -const ingestionVersioningErrorMessage = 'Versioning cannot be suspended for ' -+ 'buckets setup with Out of Band updates from a location'; +const ingestionVersioningErrorMessage = + 'Versioning cannot be suspended for ' + 'buckets setup with Out of Band updates from a location'; -const objectLockErrorMessage = 'An Object Lock configuration is present on ' + - 'this bucket, so the versioning state cannot be changed.'; +const objectLockErrorMessage = + 'An Object Lock configuration is present on ' + 'this bucket, so the versioning state cannot be changed.'; /** * Format of xml request: @@ -47,21 +47,17 @@ function _parseXML(request, log, cb) { return cb(errors.MalformedXML); } const versioningConf = result.VersioningConfiguration; - const status = versioningConf.Status ? - versioningConf.Status[0] : undefined; - const mfaDelete = versioningConf.MfaDelete ? - versioningConf.MfaDelete[0] : undefined; + const status = versioningConf.Status ? versioningConf.Status[0] : undefined; + const mfaDelete = versioningConf.MfaDelete ? versioningConf.MfaDelete[0] : undefined; const validStatuses = ['Enabled', 'Suspended']; const validMfaDeletes = [undefined, 'Enabled', 'Disabled']; - if (validStatuses.indexOf(status) < 0 || - validMfaDeletes.indexOf(mfaDelete) < 0) { + if (validStatuses.indexOf(status) < 0 || validMfaDeletes.indexOf(mfaDelete) < 0) { log.debug('illegal versioning configuration'); return cb(errors.IllegalVersioningConfigurationException); } if (versioningConf && mfaDelete === 'Enabled') { log.debug('mfa deletion is not implemented'); - return cb(errorInstances.NotImplemented - .customizeDescription('MFA Deletion is not supported yet.')); + return cb(errorInstances.NotImplemented.customizeDescription('MFA Deletion is not supported yet.')); } return process.nextTick(() => cb(null)); }); @@ -103,90 +99,89 @@ function bucketPutVersioning(authInfo, request, log, callback) { requestType: request.apiMethods || 'bucketPutVersioning', request, }; - return waterfall([ - next => _parseXML(request, log, next), - next => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, - (err, bucket) => next(err, bucket)), // ignore extra null object, - (bucket, next) => parseString(request.post, (err, result) => { - // just for linting; there should not be any parsing error here - if (err) { - return next(err, bucket); - } - // prevent enabling versioning on an nfs exported bucket - if (bucket.isNFS()) { - const error = new Error(); - error.code = 'NFSBUCKET'; - return next(error); - } - // _checkBackendVersioningImplemented returns false if versioning - // is not implemented on the bucket backend - if (!_checkBackendVersioningImplemented(bucket)) { - log.debug(externalVersioningErrorMessage, - { method: 'bucketPutVersioning', - error: errors.NotImplemented }); - const error = errorInstances.NotImplemented.customizeDescription( - externalVersioningErrorMessage); - return next(error, bucket); - } - const versioningConfiguration = {}; - if (result.VersioningConfiguration.Status) { - versioningConfiguration.Status = - result.VersioningConfiguration.Status[0]; - } - if (result.VersioningConfiguration.MfaDelete) { - versioningConfiguration.MfaDelete = - result.VersioningConfiguration.MfaDelete[0]; - } - // the configuration has been checked before - return next(null, bucket, versioningConfiguration); - }), - (bucket, versioningConfiguration, next) => { - // check if replication is enabled if versioning is being suspended - const replicationConfig = bucket.getReplicationConfiguration(); - const isIngestionBucket = bucket.isIngestionBucket && bucket.isIngestionBucket(); - const invalidAction = - versioningConfiguration.Status === 'Suspended' - && (isIngestionBucket || replicationConfig?.rules?.some(r => r.enabled)); - if (invalidAction) { - const errorMsg = isIngestionBucket ? - ingestionVersioningErrorMessage : replicationVersioningErrorMessage; - next(errorInstances.InvalidBucketState - .customizeDescription(errorMsg)); - return; + return waterfall( + [ + next => _parseXML(request, log, next), + next => + standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => + next(err, bucket), + ), // ignore extra null object, + (bucket, next) => + parseString(request.post, (err, result) => { + // just for linting; there should not be any parsing error here + if (err) { + return next(err, bucket); + } + // prevent enabling versioning on an nfs exported bucket + if (bucket.isNFS()) { + const error = new Error(); + error.code = 'NFSBUCKET'; + return next(error); + } + // _checkBackendVersioningImplemented returns false if versioning + // is not implemented on the bucket backend + if (!_checkBackendVersioningImplemented(bucket)) { + log.debug(externalVersioningErrorMessage, { + method: 'bucketPutVersioning', + error: errors.NotImplemented, + }); + const error = + errorInstances.NotImplemented.customizeDescription(externalVersioningErrorMessage); + return next(error, bucket); + } + const versioningConfiguration = {}; + if (result.VersioningConfiguration.Status) { + versioningConfiguration.Status = result.VersioningConfiguration.Status[0]; + } + if (result.VersioningConfiguration.MfaDelete) { + versioningConfiguration.MfaDelete = result.VersioningConfiguration.MfaDelete[0]; + } + // the configuration has been checked before + return next(null, bucket, versioningConfiguration); + }), + (bucket, versioningConfiguration, next) => { + // check if replication is enabled if versioning is being suspended + const replicationConfig = bucket.getReplicationConfiguration(); + const isIngestionBucket = bucket.isIngestionBucket && bucket.isIngestionBucket(); + const invalidAction = + versioningConfiguration.Status === 'Suspended' && + (isIngestionBucket || replicationConfig?.rules?.some(r => r.enabled)); + if (invalidAction) { + const errorMsg = isIngestionBucket + ? ingestionVersioningErrorMessage + : replicationVersioningErrorMessage; + next(errorInstances.InvalidBucketState.customizeDescription(errorMsg)); + return; + } + const objectLockEnabled = bucket.isObjectLockEnabled(); + if (objectLockEnabled) { + next(errorInstances.InvalidBucketState.customizeDescription(objectLockErrorMessage)); + return; + } + bucket.setVersioningConfiguration(versioningConfiguration); + // TODO all metadata updates of bucket should be using CAS + metadata.updateBucket(bucket.getName(), bucket, log, err => next(err, bucket)); + }, + ], + (err, bucket) => { + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + if (err && err.code === 'NFSBUCKET') { + log.trace('skipping versioning for nfs exported bucket'); + return callback(null, corsHeaders); } - const objectLockEnabled = bucket.isObjectLockEnabled(); - if (objectLockEnabled) { - next(errorInstances.InvalidBucketState - .customizeDescription(objectLockErrorMessage)); - return; + if (err) { + log.trace('error processing request', { error: err, method: 'bucketPutVersioning' }); + monitoring.promMetrics('PUT', bucketName, err.code, 'putBucketVersioning'); + } else { + pushMetric('putBucketVersioning', log, { + authInfo, + bucket: bucketName, + }); + monitoring.promMetrics('PUT', bucketName, '200', 'putBucketVersioning'); } - bucket.setVersioningConfiguration(versioningConfiguration); - // TODO all metadata updates of bucket should be using CAS - metadata.updateBucket(bucket.getName(), bucket, log, err => - next(err, bucket)); + return callback(err, corsHeaders); }, - ], (err, bucket) => { - const corsHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); - if (err && err.code === 'NFSBUCKET') { - log.trace('skipping versioning for nfs exported bucket'); - return callback(null, corsHeaders); - } - if (err) { - log.trace('error processing request', { error: err, - method: 'bucketPutVersioning' }); - monitoring.promMetrics( - 'PUT', bucketName, err.code, 'putBucketVersioning'); - } else { - pushMetric('putBucketVersioning', log, { - authInfo, - bucket: bucketName, - }); - monitoring.promMetrics( - 'PUT', bucketName, '200', 'putBucketVersioning'); - } - return callback(err, corsHeaders); - }); + ); } module.exports = bucketPutVersioning; diff --git a/lib/api/bucketPutWebsite.js b/lib/api/bucketPutWebsite.js index b7110a8a11..7fd1374d6c 100644 --- a/lib/api/bucketPutWebsite.js +++ b/lib/api/bucketPutWebsite.js @@ -34,42 +34,45 @@ function bucketPutWebsite(authInfo, request, log, callback) { return callback(errors.MissingRequestBodyError); } - return async.waterfall([ - next => { - log.trace('parsing website configuration'); - return parseWebsiteConfigXml(request.post, log, next); - }, - (config, next) => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, - (err, bucket) => { - const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); - if (err) { - monitoring.promMetrics('PUT', bucketName, err.code, METRICS_ACTION); - if (err?.is?.AccessDenied) { - return next(err, corsHeaders); + return async.waterfall( + [ + next => { + log.trace('parsing website configuration'); + return parseWebsiteConfigXml(request.post, log, next); + }, + (config, next) => + standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => { + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + if (err) { + monitoring.promMetrics('PUT', bucketName, err.code, METRICS_ACTION); + if (err?.is?.AccessDenied) { + return next(err, corsHeaders); + } + return next(err); } - return next(err); - } - return next(null, bucket, config, corsHeaders); - }), - (bucket, config, corsHeaders, next) => { - log.trace('updating bucket website configuration in metadata'); - bucket.setWebsiteConfiguration(config); - return metadata.updateBucket(bucketName, bucket, log, err => { - next(err, corsHeaders); - }); - } - ], (err, corsHeaders) => { - if (err) { - log.trace('error processing request', { error: err, method: REQUEST_TYPE }); - monitoring.promMetrics('PUT', bucketName, err.code, METRICS_ACTION); - return callback(err, corsHeaders); - } + return next(null, bucket, config, corsHeaders); + }), + (bucket, config, corsHeaders, next) => { + log.trace('updating bucket website configuration in metadata'); + bucket.setWebsiteConfiguration(config); + return metadata.updateBucket(bucketName, bucket, log, err => { + next(err, corsHeaders); + }); + }, + ], + (err, corsHeaders) => { + if (err) { + log.trace('error processing request', { error: err, method: REQUEST_TYPE }); + monitoring.promMetrics('PUT', bucketName, err.code, METRICS_ACTION); + return callback(err, corsHeaders); + } - pushMetric(METRICS_ACTION, log, { authInfo, bucket: bucketName }); - monitoring.promMetrics('PUT', bucketName, '200', METRICS_ACTION); - return callback(null, corsHeaders); - }); + pushMetric(METRICS_ACTION, log, { authInfo, bucket: bucketName }); + monitoring.promMetrics('PUT', bucketName, '200', METRICS_ACTION); + return callback(null, corsHeaders); + }, + ); } module.exports = bucketPutWebsite; diff --git a/lib/api/completeMultipartUpload.js b/lib/api/completeMultipartUpload.js index 83df03fbe8..581344223b 100644 --- a/lib/api/completeMultipartUpload.js +++ b/lib/api/completeMultipartUpload.js @@ -10,17 +10,18 @@ const { data } = require('../data/wrapper'); const collectCorsHeaders = require('../utilities/collectCorsHeaders'); const constants = require('../../constants'); const { config } = require('../Config'); -const { versioningPreprocessing, checkQueryVersionId, decodeVID, overwritingVersioning } - = require('./apiUtils/object/versioning'); +const { + versioningPreprocessing, + checkQueryVersionId, + decodeVID, + overwritingVersioning, +} = require('./apiUtils/object/versioning'); const services = require('../services'); const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUtils'); -const locationConstraintCheck - = require('./apiUtils/object/locationConstraintCheck'); +const locationConstraintCheck = require('./apiUtils/object/locationConstraintCheck'); const { skipMpuPartProcessing } = storage.data.external.backendUtils; -const { validateAndFilterMpuParts, generateMpuPartStorageInfo } = - s3middleware.processMpuParts; -const locationKeysHaveChanged - = require('./apiUtils/object/locationKeysHaveChanged'); +const { validateAndFilterMpuParts, generateMpuPartStorageInfo } = s3middleware.processMpuParts; +const locationKeysHaveChanged = require('./apiUtils/object/locationKeysHaveChanged'); const { setExpirationHeaders } = require('./apiUtils/object/expirationHeaders'); const { validatePutVersionId } = require('./apiUtils/object/coldStorage'); const { validateQuotas } = require('./apiUtils/quotas/quotaUtils'); @@ -49,8 +50,7 @@ const REPLICATION_ACTION = 'MPU'; */ - - /* +/* Format of xml response: { - if (err || !result || !result.CompleteMultipartUpload - || !result.CompleteMultipartUpload.Part) { + if (err || !result || !result.CompleteMultipartUpload || !result.CompleteMultipartUpload.Part) { return next(errors.MalformedXML); } const jsonList = result.CompleteMultipartUpload; @@ -126,288 +125,472 @@ function completeMultipartUpload(authInfo, request, log, callback) { }); } - return async.waterfall([ - function validateDestBucket(next) { - const metadataValParams = { - objectKey, - authInfo, - bucketName, - // Required permissions for this action - // at the destinationBucket level are same as objectPut - requestType: request.apiMethods || 'completeMultipartUpload', - versionId, - request, - }; - standardMetadataValidateBucketAndObj(metadataValParams, request.actionImplicitDenies, log, next); - }, - function validateMultipart(destBucket, objMD, next) { - if (objMD) { - oldByteLength = objMD['content-length']; - } - - if (isPutVersion) { - const error = validatePutVersionId(objMD, putVersionId, log); - if (error) { - return next(error, destBucket); + return async.waterfall( + [ + function validateDestBucket(next) { + const metadataValParams = { + objectKey, + authInfo, + bucketName, + // Required permissions for this action + // at the destinationBucket level are same as objectPut + requestType: request.apiMethods || 'completeMultipartUpload', + versionId, + request, + }; + standardMetadataValidateBucketAndObj(metadataValParams, request.actionImplicitDenies, log, next); + }, + function validateMultipart(destBucket, objMD, next) { + if (objMD) { + oldByteLength = objMD['content-length']; } - } - return services.metadataValidateMultipart(metadataValParams, - (err, mpuBucket, mpuOverview, storedMetadata) => { - if (err) { - log.error('error validating request', { error: err }); - return next(err, destBucket); - } - return next(null, destBucket, objMD, mpuBucket, - storedMetadata); - }); - }, - function parsePartsList(destBucket, objMD, mpuBucket, - storedMetadata, next) { - const location = storedMetadata.controllingLocationConstraint; - // BACKWARD: Remove to remove the old splitter - if (mpuBucket.getMdBucketModelVersion() < 2) { - splitter = constants.oldSplitter; - } - // Reconstruct mpuOverviewKey to point to metadata - // originally stored when mpu initiated - const mpuOverviewKey = - `overview${splitter}${objectKey}${splitter}${uploadId}`; - if (request.post) { - return parseXml(request.post, (err, jsonList) => { - if (err) { - log.error('error parsing XML', { error: err }); - return next(err, destBucket); + if (isPutVersion) { + const error = validatePutVersionId(objMD, putVersionId, log); + if (error) { + return next(error, destBucket); } - return next(null, destBucket, objMD, mpuBucket, - jsonList, storedMetadata, location, mpuOverviewKey); - }); - } - return next(errors.MalformedXML, destBucket); - }, - function markOverviewForCompletion(destBucket, objMD, mpuBucket, jsonList, - storedMetadata, location, mpuOverviewKey, next) { - return services.metadataMarkMPObjectForCompletion({ - bucketName: mpuBucket.getName(), - objectKey, - uploadId, - splitter, + } + + return services.metadataValidateMultipart( + metadataValParams, + (err, mpuBucket, mpuOverview, storedMetadata) => { + if (err) { + log.error('error validating request', { error: err }); + return next(err, destBucket); + } + return next(null, destBucket, objMD, mpuBucket, storedMetadata); + }, + ); + }, + function parsePartsList(destBucket, objMD, mpuBucket, storedMetadata, next) { + const location = storedMetadata.controllingLocationConstraint; + // BACKWARD: Remove to remove the old splitter + if (mpuBucket.getMdBucketModelVersion() < 2) { + splitter = constants.oldSplitter; + } + // Reconstruct mpuOverviewKey to point to metadata + // originally stored when mpu initiated + const mpuOverviewKey = `overview${splitter}${objectKey}${splitter}${uploadId}`; + if (request.post) { + return parseXml(request.post, (err, jsonList) => { + if (err) { + log.error('error parsing XML', { error: err }); + return next(err, destBucket); + } + return next( + null, + destBucket, + objMD, + mpuBucket, + jsonList, + storedMetadata, + location, + mpuOverviewKey, + ); + }); + } + return next(errors.MalformedXML, destBucket); + }, + function markOverviewForCompletion( + destBucket, + objMD, + mpuBucket, + jsonList, storedMetadata, - }, log, err => { - if (err) { - log.error('error marking MPU object for completion', { + location, + mpuOverviewKey, + next, + ) { + return services.metadataMarkMPObjectForCompletion( + { bucketName: mpuBucket.getName(), objectKey, uploadId, - error: err, - }); - return next(err); - } - return next(null, destBucket, objMD, mpuBucket, - jsonList, storedMetadata, location, mpuOverviewKey); - }); - }, - function retrieveParts(destBucket, objMD, mpuBucket, jsonList, - storedMetadata, location, mpuOverviewKey, next) { - return services.getMPUparts(mpuBucket.getName(), uploadId, log, - (err, result) => { + splitter, + storedMetadata, + }, + log, + err => { + if (err) { + log.error('error marking MPU object for completion', { + bucketName: mpuBucket.getName(), + objectKey, + uploadId, + error: err, + }); + return next(err); + } + return next( + null, + destBucket, + objMD, + mpuBucket, + jsonList, + storedMetadata, + location, + mpuOverviewKey, + ); + }, + ); + }, + function retrieveParts( + destBucket, + objMD, + mpuBucket, + jsonList, + storedMetadata, + location, + mpuOverviewKey, + next, + ) { + return services.getMPUparts(mpuBucket.getName(), uploadId, log, (err, result) => { if (err) { log.error('error getting parts', { error: err }); return next(err, destBucket); } const storedParts = result.Contents; const totalMPUSize = storedParts.reduce((acc, part) => acc + part.value.Size, 0); - return next(null, destBucket, objMD, mpuBucket, storedParts, - jsonList, storedMetadata, location, mpuOverviewKey, totalMPUSize); + return next( + null, + destBucket, + objMD, + mpuBucket, + storedParts, + jsonList, + storedMetadata, + location, + mpuOverviewKey, + totalMPUSize, + ); }); - }, - function completeExternalMpu(destBucket, objMD, mpuBucket, storedParts, - jsonList, storedMetadata, location, mpuOverviewKey, totalMPUSize, next) { - const mdInfo = { storedParts, mpuOverviewKey, splitter }; - const mpuInfo = - { objectKey, uploadId, jsonList, bucketName, destBucket }; - const originalIdentityImpDenies = request.actionImplicitDenies; - // eslint-disable-next-line no-param-reassign - delete request.actionImplicitDenies; - return data.completeMPU(request, mpuInfo, mdInfo, location, - null, null, null, locationConstraintCheck, log, - (err, completeObjData) => { + }, + function completeExternalMpu( + destBucket, + objMD, + mpuBucket, + storedParts, + jsonList, + storedMetadata, + location, + mpuOverviewKey, + totalMPUSize, + next, + ) { + const mdInfo = { storedParts, mpuOverviewKey, splitter }; + const mpuInfo = { objectKey, uploadId, jsonList, bucketName, destBucket }; + const originalIdentityImpDenies = request.actionImplicitDenies; // eslint-disable-next-line no-param-reassign - request.actionImplicitDenies = originalIdentityImpDenies; - if (err) { - log.error('error completing MPU externally', { error: err }); - return next(err, destBucket); - } - // if mpu not handled externally, completeObjData will be null - return next(null, destBucket, objMD, mpuBucket, storedParts, - jsonList, storedMetadata, completeObjData, mpuOverviewKey, - totalMPUSize); - }); - }, - function validateAndFilterParts(destBucket, objMD, mpuBucket, - storedParts, jsonList, storedMetadata, completeObjData, mpuOverviewKey, - totalMPUSize, next) { - if (completeObjData) { - return next(null, destBucket, objMD, mpuBucket, storedParts, - jsonList, storedMetadata, completeObjData, mpuOverviewKey, - completeObjData.filteredPartsObj, totalMPUSize); - } - const filteredPartsObj = validateAndFilterMpuParts(storedParts, - jsonList, mpuOverviewKey, splitter, log); - if (filteredPartsObj.error) { - return next(filteredPartsObj.error, destBucket); - } - return next(null, destBucket, objMD, mpuBucket, storedParts, - jsonList, storedMetadata, completeObjData, mpuOverviewKey, - filteredPartsObj, totalMPUSize); - }, - function processParts(destBucket, objMD, mpuBucket, storedParts, - jsonList, storedMetadata, completeObjData, mpuOverviewKey, - filteredPartsObj, totalMPUSize, next) { - // if mpu was completed on backend that stored mpu MD externally, - // skip MD processing steps - if (completeObjData && skipMpuPartProcessing(completeObjData)) { - const dataLocations = [ - { - key: completeObjData.key, - size: completeObjData.contentLength, - start: 0, - dataStoreVersionId: completeObjData.dataStoreVersionId, - dataStoreName: storedMetadata.dataStoreName, - dataStoreETag: completeObjData.eTag, - dataStoreType: completeObjData.dataStoreType, + delete request.actionImplicitDenies; + return data.completeMPU( + request, + mpuInfo, + mdInfo, + location, + null, + null, + null, + locationConstraintCheck, + log, + (err, completeObjData) => { + // eslint-disable-next-line no-param-reassign + request.actionImplicitDenies = originalIdentityImpDenies; + if (err) { + log.error('error completing MPU externally', { error: err }); + return next(err, destBucket); + } + // if mpu not handled externally, completeObjData will be null + return next( + null, + destBucket, + objMD, + mpuBucket, + storedParts, + jsonList, + storedMetadata, + completeObjData, + mpuOverviewKey, + totalMPUSize, + ); }, - ]; - const calculatedSize = completeObjData.contentLength; - return next(null, destBucket, objMD, mpuBucket, storedMetadata, - completeObjData.eTag, calculatedSize, dataLocations, - [mpuOverviewKey], null, completeObjData, totalMPUSize); - } - - const partsInfo = - generateMpuPartStorageInfo(filteredPartsObj.partList); - if (partsInfo.error) { - return next(partsInfo.error, destBucket); - } - const { keysToDelete, extraPartLocations } = filteredPartsObj; - const { aggregateETag, dataLocations, calculatedSize } = partsInfo; + ); + }, + function validateAndFilterParts( + destBucket, + objMD, + mpuBucket, + storedParts, + jsonList, + storedMetadata, + completeObjData, + mpuOverviewKey, + totalMPUSize, + next, + ) { + if (completeObjData) { + return next( + null, + destBucket, + objMD, + mpuBucket, + storedParts, + jsonList, + storedMetadata, + completeObjData, + mpuOverviewKey, + completeObjData.filteredPartsObj, + totalMPUSize, + ); + } + const filteredPartsObj = validateAndFilterMpuParts( + storedParts, + jsonList, + mpuOverviewKey, + splitter, + log, + ); + if (filteredPartsObj.error) { + return next(filteredPartsObj.error, destBucket); + } + return next( + null, + destBucket, + objMD, + mpuBucket, + storedParts, + jsonList, + storedMetadata, + completeObjData, + mpuOverviewKey, + filteredPartsObj, + totalMPUSize, + ); + }, + function processParts( + destBucket, + objMD, + mpuBucket, + storedParts, + jsonList, + storedMetadata, + completeObjData, + mpuOverviewKey, + filteredPartsObj, + totalMPUSize, + next, + ) { + // if mpu was completed on backend that stored mpu MD externally, + // skip MD processing steps + if (completeObjData && skipMpuPartProcessing(completeObjData)) { + const dataLocations = [ + { + key: completeObjData.key, + size: completeObjData.contentLength, + start: 0, + dataStoreVersionId: completeObjData.dataStoreVersionId, + dataStoreName: storedMetadata.dataStoreName, + dataStoreETag: completeObjData.eTag, + dataStoreType: completeObjData.dataStoreType, + }, + ]; + const calculatedSize = completeObjData.contentLength; + return next( + null, + destBucket, + objMD, + mpuBucket, + storedMetadata, + completeObjData.eTag, + calculatedSize, + dataLocations, + [mpuOverviewKey], + null, + completeObjData, + totalMPUSize, + ); + } - if (completeObjData) { - const dataLocations = [ - { - key: completeObjData.key, - size: calculatedSize, - start: 0, - dataStoreName: storedMetadata.dataStoreName, - dataStoreETag: aggregateETag, - dataStoreType: completeObjData.dataStoreType, - }, + const partsInfo = generateMpuPartStorageInfo(filteredPartsObj.partList); + if (partsInfo.error) { + return next(partsInfo.error, destBucket); + } + const { keysToDelete, extraPartLocations } = filteredPartsObj; + const { aggregateETag, dataLocations, calculatedSize } = partsInfo; + + if (completeObjData) { + const dataLocations = [ + { + key: completeObjData.key, + size: calculatedSize, + start: 0, + dataStoreName: storedMetadata.dataStoreName, + dataStoreETag: aggregateETag, + dataStoreType: completeObjData.dataStoreType, + }, + ]; + return next( + null, + destBucket, + objMD, + mpuBucket, + storedMetadata, + aggregateETag, + calculatedSize, + dataLocations, + keysToDelete, + extraPartLocations, + completeObjData, + totalMPUSize, + ); + } + return next( + null, + destBucket, + objMD, + mpuBucket, + storedMetadata, + aggregateETag, + calculatedSize, + dataLocations, + keysToDelete, + extraPartLocations, + null, + totalMPUSize, + ); + }, + function prepForStoring( + destBucket, + objMD, + mpuBucket, + storedMetadata, + aggregateETag, + calculatedSize, + dataLocations, + keysToDelete, + extraPartLocations, + completeObjData, + totalMPUSize, + next, + ) { + // Store full object size for server access logs + if (request.serverAccessLog) { + // eslint-disable-next-line no-param-reassign + request.serverAccessLog.objectSize = calculatedSize; + } + const metaHeaders = {}; + const keysNotNeeded = [ + 'initiator', + 'partLocations', + 'key', + 'initiated', + 'uploadId', + 'content-type', + 'expires', + 'eventualStorageBucket', + 'dataStoreName', ]; - return next(null, destBucket, objMD, mpuBucket, storedMetadata, - aggregateETag, calculatedSize, dataLocations, keysToDelete, - extraPartLocations, completeObjData, totalMPUSize); - } - return next(null, destBucket, objMD, mpuBucket, storedMetadata, - aggregateETag, calculatedSize, dataLocations, keysToDelete, - extraPartLocations, null, totalMPUSize); - }, - function prepForStoring(destBucket, objMD, mpuBucket, storedMetadata, - aggregateETag, calculatedSize, dataLocations, keysToDelete, - extraPartLocations, completeObjData, totalMPUSize, next) { - // Store full object size for server access logs - if (request.serverAccessLog) { - // eslint-disable-next-line no-param-reassign - request.serverAccessLog.objectSize = calculatedSize; - } - const metaHeaders = {}; - const keysNotNeeded = - ['initiator', 'partLocations', 'key', - 'initiated', 'uploadId', 'content-type', 'expires', - 'eventualStorageBucket', 'dataStoreName']; - const metadataKeysToPull = - Object.keys(storedMetadata).filter(item => - keysNotNeeded.indexOf(item) === -1); - metadataKeysToPull.forEach(item => { - metaHeaders[item] = storedMetadata[item]; - }); - - const droppedMPUSize = totalMPUSize - calculatedSize; + const metadataKeysToPull = Object.keys(storedMetadata).filter( + item => keysNotNeeded.indexOf(item) === -1, + ); + metadataKeysToPull.forEach(item => { + metaHeaders[item] = storedMetadata[item]; + }); - const metaStoreParams = { - authInfo, - objectKey, - metaHeaders, - uploadId, - dataStoreName: storedMetadata.dataStoreName, - contentType: storedMetadata['content-type'], - cacheControl: storedMetadata['cache-control'], - contentDisposition: storedMetadata['content-disposition'], - contentEncoding: storedMetadata['content-encoding'], - expires: storedMetadata.expires, - contentMD5: aggregateETag, - size: calculatedSize, - multipart: true, - isDeleteMarker: false, - replicationInfo: getReplicationInfo(config, - objectKey, destBucket, false, calculatedSize, REPLICATION_ACTION), - originOp: 's3:ObjectCreated:CompleteMultipartUpload', - overheadField: constants.overheadField, - log, - }; - // If key already exists - if (objMD) { - // Re-use creation-time if we can - if (objMD['creation-time']) { - metaStoreParams.creationTime = objMD['creation-time']; - // Otherwise fallback to last-modified + const droppedMPUSize = totalMPUSize - calculatedSize; + + const metaStoreParams = { + authInfo, + objectKey, + metaHeaders, + uploadId, + dataStoreName: storedMetadata.dataStoreName, + contentType: storedMetadata['content-type'], + cacheControl: storedMetadata['cache-control'], + contentDisposition: storedMetadata['content-disposition'], + contentEncoding: storedMetadata['content-encoding'], + expires: storedMetadata.expires, + contentMD5: aggregateETag, + size: calculatedSize, + multipart: true, + isDeleteMarker: false, + replicationInfo: getReplicationInfo( + config, + objectKey, + destBucket, + false, + calculatedSize, + REPLICATION_ACTION, + ), + originOp: 's3:ObjectCreated:CompleteMultipartUpload', + overheadField: constants.overheadField, + log, + }; + // If key already exists + if (objMD) { + // Re-use creation-time if we can + if (objMD['creation-time']) { + metaStoreParams.creationTime = objMD['creation-time']; + // Otherwise fallback to last-modified + } else { + metaStoreParams.creationTime = objMD['last-modified']; + } + // If its a new key, create a new timestamp } else { - metaStoreParams.creationTime = objMD['last-modified']; + metaStoreParams.creationTime = new Date().toJSON(); + } + if (storedMetadata['x-amz-tagging']) { + metaStoreParams.tagging = storedMetadata['x-amz-tagging']; + } + if (storedMetadata.retentionMode && storedMetadata.retentionDate) { + metaStoreParams.retentionMode = storedMetadata.retentionMode; + metaStoreParams.retentionDate = storedMetadata.retentionDate; + } + if (storedMetadata.legalHold) { + metaStoreParams.legalHold = storedMetadata.legalHold; + } + const serverSideEncryption = storedMetadata['x-amz-server-side-encryption']; + let pseudoCipherBundle = null; + if (serverSideEncryption) { + const kmsKey = storedMetadata['x-amz-server-side-encryption-aws-kms-key-id']; + pseudoCipherBundle = { + algorithm: serverSideEncryption, + masterKeyId: kmsKey, + }; + setSSEHeaders(responseHeaders, serverSideEncryption, kmsKey); } - // If its a new key, create a new timestamp - } else { - metaStoreParams.creationTime = new Date().toJSON(); - } - if (storedMetadata['x-amz-tagging']) { - metaStoreParams.tagging = storedMetadata['x-amz-tagging']; - } - if (storedMetadata.retentionMode && storedMetadata.retentionDate) { - metaStoreParams.retentionMode = storedMetadata.retentionMode; - metaStoreParams.retentionDate = storedMetadata.retentionDate; - } - if (storedMetadata.legalHold) { - metaStoreParams.legalHold = storedMetadata.legalHold; - } - const serverSideEncryption = storedMetadata['x-amz-server-side-encryption']; - let pseudoCipherBundle = null; - if (serverSideEncryption) { - const kmsKey = storedMetadata['x-amz-server-side-encryption-aws-kms-key-id']; - pseudoCipherBundle = { - algorithm: serverSideEncryption, - masterKeyId: kmsKey, - }; - setSSEHeaders(responseHeaders, serverSideEncryption, kmsKey); - } - if (authInfo.getCanonicalID() !== destBucket.getOwner()) { - metaStoreParams.bucketOwnerId = destBucket.getOwner(); - } + if (authInfo.getCanonicalID() !== destBucket.getOwner()) { + metaStoreParams.bucketOwnerId = destBucket.getOwner(); + } - // if x-scal-s3-version-id header is specified, we overwrite the object/version metadata. - if (isPutVersion) { - const options = overwritingVersioning(objMD, metaStoreParams); - return process.nextTick(() => next(null, destBucket, dataLocations, - metaStoreParams, mpuBucket, keysToDelete, aggregateETag, - objMD, extraPartLocations, pseudoCipherBundle, - completeObjData, options, droppedMPUSize)); - } + // if x-scal-s3-version-id header is specified, we overwrite the object/version metadata. + if (isPutVersion) { + const options = overwritingVersioning(objMD, metaStoreParams); + return process.nextTick(() => + next( + null, + destBucket, + dataLocations, + metaStoreParams, + mpuBucket, + keysToDelete, + aggregateETag, + objMD, + extraPartLocations, + pseudoCipherBundle, + completeObjData, + options, + droppedMPUSize, + ), + ); + } - if (!destBucket.isVersioningEnabled() && objMD?.archive?.archiveInfo) { - // Ensure we trigger a "delete" event in the oplog for the previously archived object - metaStoreParams.needOplogUpdate = 's3:ReplaceArchivedObject'; - } + if (!destBucket.isVersioningEnabled() && objMD?.archive?.archiveInfo) { + // Ensure we trigger a "delete" event in the oplog for the previously archived object + metaStoreParams.needOplogUpdate = 's3:ReplaceArchivedObject'; + } - return versioningPreprocessing(bucketName, - destBucket, objectKey, objMD, log, (err, options) => { + return versioningPreprocessing(bucketName, destBucket, objectKey, objMD, log, (err, options) => { if (err) { // TODO: check AWS error when user requested a specific // version before any versions have been put @@ -430,206 +613,287 @@ function completeMultipartUpload(authInfo, request, log, callback) { } } - return next(null, destBucket, dataLocations, - metaStoreParams, mpuBucket, keysToDelete, aggregateETag, - objMD, extraPartLocations, pseudoCipherBundle, - completeObjData, options, droppedMPUSize); + return next( + null, + destBucket, + dataLocations, + metaStoreParams, + mpuBucket, + keysToDelete, + aggregateETag, + objMD, + extraPartLocations, + pseudoCipherBundle, + completeObjData, + options, + droppedMPUSize, + ); }); - }, - function storeAsNewObj(destinationBucket, dataLocations, - metaStoreParams, mpuBucket, keysToDelete, aggregateETag, objMD, - extraPartLocations, pseudoCipherBundle, - completeObjData, options, droppedMPUSize, next) { - const dataToDelete = options.dataToDelete; - /* eslint-disable no-param-reassign */ - metaStoreParams.versionId = options.versionId; - metaStoreParams.versioning = options.versioning; - metaStoreParams.isNull = options.isNull; - metaStoreParams.deleteNullKey = options.deleteNullKey; - if (options.extraMD) { - Object.assign(metaStoreParams, options.extraMD); - } - /* eslint-enable no-param-reassign */ - - // For external backends (where completeObjData is not - // null), the backend key does not change for new versions - // of the same object (or rewrites for nonversioned - // buckets), hence the deduplication sanity check does not - // make sense for external backends. - if (objMD && !completeObjData) { - // An object with the same key already exists, check - // if it has been created by the same MPU upload by - // checking if any of its internal location keys match - // the new keys. In such case, it must be a duplicate - // from a retry of a previous failed completion - // attempt, hence do the following: - // - // - skip writing the new metadata key to avoid - // creating a new version pointing to the same data - // keys - // - // - skip old data locations deletion since the old - // data location keys overlap the new ones (in - // principle they should be fully identical as there - // is no reuse of previous versions' data keys in - // the normal process) - note that the previous - // failed completion attempt may have left orphan - // data keys but we lost track of them so we cannot - // delete them now - // - // - proceed to the deletion of overview and part - // metadata keys, which are likely to have failed in - // the previous MPU completion attempt - // - if (!locationKeysHaveChanged(objMD.location, dataLocations)) { - log.info('MPU complete request replay detected', { - method: 'completeMultipartUpload.storeAsNewObj', - bucketName: destinationBucket.getName(), - objectKey: metaStoreParams.objectKey, - uploadId: metaStoreParams.uploadId, - }); - return next(null, mpuBucket, keysToDelete, aggregateETag, - extraPartLocations, destinationBucket, - // pass the original version ID as generatedVersionId - objMD.versionId, droppedMPUSize); + }, + function storeAsNewObj( + destinationBucket, + dataLocations, + metaStoreParams, + mpuBucket, + keysToDelete, + aggregateETag, + objMD, + extraPartLocations, + pseudoCipherBundle, + completeObjData, + options, + droppedMPUSize, + next, + ) { + const dataToDelete = options.dataToDelete; + /* eslint-disable no-param-reassign */ + metaStoreParams.versionId = options.versionId; + metaStoreParams.versioning = options.versioning; + metaStoreParams.isNull = options.isNull; + metaStoreParams.deleteNullKey = options.deleteNullKey; + if (options.extraMD) { + Object.assign(metaStoreParams, options.extraMD); } - } - return services.metadataStoreObject(destinationBucket.getName(), - dataLocations, pseudoCipherBundle, metaStoreParams, - (err, res) => { - if (err) { - log.error('error storing object metadata', { error: err }); - return next(err, destinationBucket); + /* eslint-enable no-param-reassign */ + + // For external backends (where completeObjData is not + // null), the backend key does not change for new versions + // of the same object (or rewrites for nonversioned + // buckets), hence the deduplication sanity check does not + // make sense for external backends. + if (objMD && !completeObjData) { + // An object with the same key already exists, check + // if it has been created by the same MPU upload by + // checking if any of its internal location keys match + // the new keys. In such case, it must be a duplicate + // from a retry of a previous failed completion + // attempt, hence do the following: + // + // - skip writing the new metadata key to avoid + // creating a new version pointing to the same data + // keys + // + // - skip old data locations deletion since the old + // data location keys overlap the new ones (in + // principle they should be fully identical as there + // is no reuse of previous versions' data keys in + // the normal process) - note that the previous + // failed completion attempt may have left orphan + // data keys but we lost track of them so we cannot + // delete them now + // + // - proceed to the deletion of overview and part + // metadata keys, which are likely to have failed in + // the previous MPU completion attempt + // + if (!locationKeysHaveChanged(objMD.location, dataLocations)) { + log.info('MPU complete request replay detected', { + method: 'completeMultipartUpload.storeAsNewObj', + bucketName: destinationBucket.getName(), + objectKey: metaStoreParams.objectKey, + uploadId: metaStoreParams.uploadId, + }); + return next( + null, + mpuBucket, + keysToDelete, + aggregateETag, + extraPartLocations, + destinationBucket, + // pass the original version ID as generatedVersionId + objMD.versionId, + droppedMPUSize, + ); } + } + return services.metadataStoreObject( + destinationBucket.getName(), + dataLocations, + pseudoCipherBundle, + metaStoreParams, + (err, res) => { + if (err) { + log.error('error storing object metadata', { error: err }); + return next(err, destinationBucket); + } - setExpirationHeaders(responseHeaders, { - lifecycleConfig: destinationBucket.getLifecycleConfiguration(), - objectParams: { - key: objectKey, - date: res.lastModified, - tags: res.tags, - }, - }); + setExpirationHeaders(responseHeaders, { + lifecycleConfig: destinationBucket.getLifecycleConfiguration(), + objectParams: { + key: objectKey, + date: res.lastModified, + tags: res.tags, + }, + }); - const generatedVersionId = res ? res.versionId : undefined; - // in cases where completing mpu overwrites a previous - // null version when versioning is suspended or versioning - // is not enabled, need to delete pre-existing data - // unless the preexisting object and the completed mpu - // are on external backends - if (dataToDelete) { - const newDataStoreName = - Array.isArray(dataLocations) && dataLocations[0] ? - dataLocations[0].dataStoreName : null; - return data.batchDelete(dataToDelete, - request.method, - newDataStoreName, log, err => { + const generatedVersionId = res ? res.versionId : undefined; + // in cases where completing mpu overwrites a previous + // null version when versioning is suspended or versioning + // is not enabled, need to delete pre-existing data + // unless the preexisting object and the completed mpu + // are on external backends + if (dataToDelete) { + const newDataStoreName = + Array.isArray(dataLocations) && dataLocations[0] + ? dataLocations[0].dataStoreName + : null; + return data.batchDelete(dataToDelete, request.method, newDataStoreName, log, err => { if (err) { return next(err); } - return next(null, mpuBucket, keysToDelete, - aggregateETag, extraPartLocations, - destinationBucket, generatedVersionId, - droppedMPUSize); + return next( + null, + mpuBucket, + keysToDelete, + aggregateETag, + extraPartLocations, + destinationBucket, + generatedVersionId, + droppedMPUSize, + ); }); - } - return next(null, mpuBucket, keysToDelete, aggregateETag, - extraPartLocations, destinationBucket, - generatedVersionId, droppedMPUSize); - }); - }, - function deletePartsMetadata(mpuBucket, keysToDelete, aggregateETag, - extraPartLocations, destinationBucket, generatedVersionId, droppedMPUSize, next) { - services.batchDeleteObjectMetadata(mpuBucket.getName(), - keysToDelete, log, err => { + } + return next( + null, + mpuBucket, + keysToDelete, + aggregateETag, + extraPartLocations, + destinationBucket, + generatedVersionId, + droppedMPUSize, + ); + }, + ); + }, + function deletePartsMetadata( + mpuBucket, + keysToDelete, + aggregateETag, + extraPartLocations, + destinationBucket, + generatedVersionId, + droppedMPUSize, + next, + ) { + services.batchDeleteObjectMetadata(mpuBucket.getName(), keysToDelete, log, err => { if (err) { if (err.is?.DeleteConflict) { // DeleteConflict should trigger automatic retry // Convert to InternalError to make it retryable const customErr = errorInstances.InternalError.customizeDescription( - 'conflict deleting MPU parts metadata' + 'conflict deleting MPU parts metadata', ); - return next(customErr, extraPartLocations, - destinationBucket, aggregateETag, generatedVersionId, droppedMPUSize); + return next( + customErr, + extraPartLocations, + destinationBucket, + aggregateETag, + generatedVersionId, + droppedMPUSize, + ); } // For NoSuchKey and other errors, return them as-is // NoSuchKey is non-retryable, InternalError and others are retryable - return next(err, extraPartLocations, - destinationBucket, aggregateETag, generatedVersionId, droppedMPUSize); + return next( + err, + extraPartLocations, + destinationBucket, + aggregateETag, + generatedVersionId, + droppedMPUSize, + ); } - return next(null, extraPartLocations, - destinationBucket, aggregateETag, generatedVersionId, droppedMPUSize); - }); - }, - function batchDeleteExtraParts(extraPartLocations, destinationBucket, - aggregateETag, generatedVersionId, droppedMPUSize, next) { - if (extraPartLocations && extraPartLocations.length > 0) { - return data.batchDelete(extraPartLocations, request.method, null, log, err => { - if (err) { - // Extra part deletion failure should not fail the operation - // The S3 object was created successfully and MPU metadata was cleaned up - // Orphaned extra parts are acceptable since the main operation succeeded - log.warn('failed to delete extra parts, keeping orphan but returning success', { - method: 'completeMultipartUpload', - extraPartLocationsCount: extraPartLocations.length, - error: err, - }); - } - return next(null, destinationBucket, aggregateETag, - generatedVersionId, droppedMPUSize); + return next( + null, + extraPartLocations, + destinationBucket, + aggregateETag, + generatedVersionId, + droppedMPUSize, + ); }); + }, + function batchDeleteExtraParts( + extraPartLocations, + destinationBucket, + aggregateETag, + generatedVersionId, + droppedMPUSize, + next, + ) { + if (extraPartLocations && extraPartLocations.length > 0) { + return data.batchDelete(extraPartLocations, request.method, null, log, err => { + if (err) { + // Extra part deletion failure should not fail the operation + // The S3 object was created successfully and MPU metadata was cleaned up + // Orphaned extra parts are acceptable since the main operation succeeded + log.warn('failed to delete extra parts, keeping orphan but returning success', { + method: 'completeMultipartUpload', + extraPartLocationsCount: extraPartLocations.length, + error: err, + }); + } + return next(null, destinationBucket, aggregateETag, generatedVersionId, droppedMPUSize); + }); + } + return next(null, destinationBucket, aggregateETag, generatedVersionId, droppedMPUSize); + }, + function updateQuotas(destinationBucket, aggregateETag, generatedVersionId, droppedMPUSize, next) { + return validateQuotas( + request, + destinationBucket, + request.accountQuotas, + ['objectDelete'], + 'objectDelete', + -droppedMPUSize, + false, + log, + err => { + if (err) { + // Ignore error, as the data has been deleted already: only inflight count + // has not been updated, and will be eventually consistent anyway + log.warn('failed to update inflights', { + method: 'completeMultipartUpload', + error: err, + }); + } + return next(null, destinationBucket, aggregateETag, generatedVersionId); + }, + ); + }, + ], + (err, destinationBucket, aggregateETag, generatedVersionId) => { + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, destinationBucket); + if (err) { + return callback(err, null, corsHeaders); } - return next(null, destinationBucket, aggregateETag, - generatedVersionId, droppedMPUSize); - }, - function updateQuotas(destinationBucket, aggregateETag, generatedVersionId, droppedMPUSize, next) { - return validateQuotas(request, destinationBucket, request.accountQuotas, - ['objectDelete'], 'objectDelete', -droppedMPUSize, false, log, err => { - if (err) { - // Ignore error, as the data has been deleted already: only inflight count - // has not been updated, and will be eventually consistent anyway - log.warn('failed to update inflights', { - method: 'completeMultipartUpload', - error: err, - }); - } - return next(null, destinationBucket, aggregateETag, - generatedVersionId); + if (generatedVersionId) { + corsHeaders['x-amz-version-id'] = versionIdUtils.encode(generatedVersionId); + } + Object.assign(responseHeaders, corsHeaders); + + const vcfg = destinationBucket.getVersioningConfiguration(); + const isVersionedObj = vcfg && vcfg.Status === 'Enabled'; + + xmlParams.eTag = `"${aggregateETag}"`; + const xml = convertToXml('completeMultipartUpload', xmlParams); + pushMetric('completeMultipartUpload', log, { + oldByteLength: isVersionedObj ? null : oldByteLength, + authInfo, + canonicalID: destinationBucket.getOwner(), + bucket: bucketName, + keys: [objectKey], + versionId: generatedVersionId, + numberOfObjects: !generatedVersionId && oldByteLength !== null ? 0 : 1, + location: destinationBucket.getLocationConstraint(), }); + return callback(null, xml, responseHeaders); }, - ], (err, destinationBucket, aggregateETag, generatedVersionId) => { - const corsHeaders = - collectCorsHeaders(request.headers.origin, request.method, - destinationBucket); - if (err) { - return callback(err, null, corsHeaders); - } - if (generatedVersionId) { - corsHeaders['x-amz-version-id'] = - versionIdUtils.encode(generatedVersionId); - } - Object.assign(responseHeaders, corsHeaders); - - const vcfg = destinationBucket.getVersioningConfiguration(); - const isVersionedObj = vcfg && vcfg.Status === 'Enabled'; - - xmlParams.eTag = `"${aggregateETag}"`; - const xml = convertToXml('completeMultipartUpload', xmlParams); - pushMetric('completeMultipartUpload', log, { - oldByteLength: isVersionedObj ? null : oldByteLength, - authInfo, - canonicalID: destinationBucket.getOwner(), - bucket: bucketName, - keys: [objectKey], - versionId: generatedVersionId, - numberOfObjects: !generatedVersionId && oldByteLength !== null ? 0 : 1, - location: destinationBucket.getLocationConstraint(), - }); - return callback(null, xml, responseHeaders); - }); + ); } module.exports = completeMultipartUpload; diff --git a/lib/api/corsPreflight.js b/lib/api/corsPreflight.js index 78ad92caf5..0200b019ea 100644 --- a/lib/api/corsPreflight.js +++ b/lib/api/corsPreflight.js @@ -2,42 +2,45 @@ const { errors, errorInstances } = require('arsenal'); const metadata = require('../metadata/wrapper'); const bucketShield = require('./apiUtils/bucket/bucketShield'); -const { findCorsRule, generateCorsResHeaders } - = require('./apiUtils/object/corsResponse'); +const { findCorsRule, generateCorsResHeaders } = require('./apiUtils/object/corsResponse'); // const { pushMetric } = require('../utapi/utilities'); const requestType = 'objectGet'; const customizedErrs = { corsNotEnabled: 'CORSResponse: CORS is not enabled for this bucket.', - notAllowed: 'CORSResponse: This CORS request is not allowed. ' + - 'This is usually because the evalution of Origin, request method / ' + - 'Access-Control-Request-Method or Access-Control-Request-Headers ' + - 'are not whitelisted by the resource\'s CORS spec.', + notAllowed: + 'CORSResponse: This CORS request is not allowed. ' + + 'This is usually because the evalution of Origin, request method / ' + + 'Access-Control-Request-Method or Access-Control-Request-Headers ' + + "are not whitelisted by the resource's CORS spec.", }; /** corsPreflight - handle preflight CORS requests -* @param {object} request - http request object -* @param {function} log - Werelogs request logger -* @param {function} callback - callback to respond to http request -* with either error code or 200 response -* @return {undefined} -*/ + * @param {object} request - http request object + * @param {function} log - Werelogs request logger + * @param {function} callback - callback to respond to http request + * with either error code or 200 response + * @return {undefined} + */ function corsPreflight(request, log, callback) { log.debug('processing request', { method: 'corsPreflight' }); const bucketName = request.bucketName; const corsOrigin = request.headers.origin; const corsMethod = request.headers['access-control-request-method']; - const corsHeaders = request.headers['access-control-request-headers'] ? - request.headers['access-control-request-headers'].replace(/ /g, '') - .split(',').reduce((resultArr, value) => { - // remove empty values and convert values to lowercase - if (value !== '') { - resultArr.push(value.toLowerCase()); - } - return resultArr; - }, []) : null; + const corsHeaders = request.headers['access-control-request-headers'] + ? request.headers['access-control-request-headers'] + .replace(/ /g, '') + .split(',') + .reduce((resultArr, value) => { + // remove empty values and convert values to lowercase + if (value !== '') { + resultArr.push(value.toLowerCase()); + } + return resultArr; + }, []) + : null; return metadata.getBucket(bucketName, log, (err, bucket) => { if (err) { @@ -51,8 +54,7 @@ function corsPreflight(request, log, callback) { const corsRules = bucket.getCors(); if (!corsRules) { - const err = errorInstances.AccessForbidden - .customizeDescription(customizedErrs.corsNotEnabled); + const err = errorInstances.AccessForbidden.customizeDescription(customizedErrs.corsNotEnabled); log.trace('no existing cors configuration', { error: err, method: 'corsPreflight', @@ -61,12 +63,10 @@ function corsPreflight(request, log, callback) { } log.trace('finding cors rule'); - const corsRule = findCorsRule(corsRules, corsOrigin, corsMethod, - corsHeaders); + const corsRule = findCorsRule(corsRules, corsOrigin, corsMethod, corsHeaders); if (!corsRule) { - const err = errorInstances.AccessForbidden - .customizeDescription(customizedErrs.notAllowed); + const err = errorInstances.AccessForbidden.customizeDescription(customizedErrs.notAllowed); log.trace('no matching cors rule', { error: err, method: 'corsPreflight', @@ -74,8 +74,7 @@ function corsPreflight(request, log, callback) { return callback(err); } - const resHeaders = generateCorsResHeaders(corsRule, corsOrigin, - corsMethod, corsHeaders, true); + const resHeaders = generateCorsResHeaders(corsRule, corsOrigin, corsMethod, corsHeaders, true); // TODO: add some level of metrics for non-standard API request: // pushMetric('corsPreflight', log, { bucket: bucketName }); return callback(null, resHeaders); diff --git a/lib/api/initiateMultipartUpload.js b/lib/api/initiateMultipartUpload.js index 3cc8e17f02..56082ff448 100644 --- a/lib/api/initiateMultipartUpload.js +++ b/lib/api/initiateMultipartUpload.js @@ -11,15 +11,12 @@ const { cleanUpBucket } = require('./apiUtils/bucket/bucketCreation'); const constants = require('../../constants'); const services = require('../services'); const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUtils'); -const locationConstraintCheck - = require('./apiUtils/object/locationConstraintCheck'); -const validateWebsiteHeader = require('./apiUtils/object/websiteServing') - .validateWebsiteHeader; +const locationConstraintCheck = require('./apiUtils/object/locationConstraintCheck'); +const validateWebsiteHeader = require('./apiUtils/object/websiteServing').validateWebsiteHeader; const monitoring = require('../utilities/monitoringHandler'); const { data } = require('../data/wrapper'); const applyZenkoUserMD = require('./apiUtils/object/applyZenkoUserMD'); -const { validateHeaders, compareObjectLockInformation } = - require('./apiUtils/object/objectLockHelpers'); +const { validateHeaders, compareObjectLockInformation } = require('./apiUtils/object/objectLockHelpers'); const { getObjectSSEConfiguration } = require('./apiUtils/bucket/bucketEncryption'); const { setExpirationHeaders } = require('./apiUtils/object/expirationHeaders'); const { setSSEHeaders } = require('./apiUtils/object/sseHeaders'); @@ -56,9 +53,9 @@ function initiateMultipartUpload(authInfo, request, log, callback) { const objectKey = request.objectKey; if (hasNonPrintables(objectKey)) { - return callback(errorInstances.InvalidInput.customizeDescription( - 'object keys cannot contain non-printable characters', - )); + return callback( + errorInstances.InvalidInput.customizeDescription('object keys cannot contain non-printable characters'), + ); } const keyLengthError = validateObjectKeyLength(objectKey, config.objectKeyByteLimit); @@ -72,19 +69,18 @@ function initiateMultipartUpload(authInfo, request, log, callback) { // there is the possiblity that the chosen splitter will occur in the object // name itself. To prevent this, we are restricting the creation of a // multipart upload object with a key containing the splitter. - const websiteRedirectHeader = - request.headers['x-amz-website-redirect-location']; - if (request.headers['x-amz-storage-class'] && - !constants.validStorageClasses.includes(request.headers['x-amz-storage-class'])) { + const websiteRedirectHeader = request.headers['x-amz-website-redirect-location']; + if ( + request.headers['x-amz-storage-class'] && + !constants.validStorageClasses.includes(request.headers['x-amz-storage-class']) + ) { log.trace('invalid storage-class header'); - monitoring.promMetrics('PUT', bucketName, - errorInstances.InvalidStorageClass.code, 'initiateMultipartUpload'); + monitoring.promMetrics('PUT', bucketName, errorInstances.InvalidStorageClass.code, 'initiateMultipartUpload'); return callback(errors.InvalidStorageClass); } if (!validateWebsiteHeader(websiteRedirectHeader)) { const err = errors.InvalidRedirectLocation; - log.debug('invalid x-amz-website-redirect-location' + - `value ${websiteRedirectHeader}`, { error: err }); + log.debug('invalid x-amz-website-redirect-location' + `value ${websiteRedirectHeader}`, { error: err }); return callback(err); } const metaHeaders = getMetaHeaders(request.headers); @@ -103,12 +99,10 @@ function initiateMultipartUpload(authInfo, request, log, callback) { // but after authentication so that string to sign is not impacted // This is GH Issue#89 // TODO: remove in CLDSRV-639 - const storageClassOptions = - ['standard', 'standard_ia', 'reduced_redundancy']; + const storageClassOptions = ['standard', 'standard_ia', 'reduced_redundancy']; let storageClass = 'STANDARD'; if (storageClassOptions.indexOf(request.headers['x-amz-storage-class']) > -1) { - storageClass = request.headers['x-amz-storage-class'] - .toUpperCase(); + storageClass = request.headers['x-amz-storage-class'].toUpperCase(); } const metadataValParams = { objectKey, @@ -150,8 +144,7 @@ function initiateMultipartUpload(authInfo, request, log, callback) { metadataStoreParams.tagging = tagging; } - function _getMPUBucket(destinationBucket, log, corsHeaders, - uploadId, cipherBundle, locConstraint, callback) { + function _getMPUBucket(destinationBucket, log, corsHeaders, uploadId, cipherBundle, locConstraint, callback) { const xmlParams = { bucketName, objectKey, @@ -160,56 +153,59 @@ function initiateMultipartUpload(authInfo, request, log, callback) { const xml = convertToXml('initiateMultipartUpload', xmlParams); metadataStoreParams.uploadId = uploadId; - services.getMPUBucket(destinationBucket, bucketName, log, - (err, MPUbucket) => { - if (err) { - log.trace('error getting MPUbucket', { - error: err, - }); - return callback(err); - } - // BACKWARD: Remove to remove the old splitter - if (MPUbucket.getMdBucketModelVersion() < 2) { - metadataStoreParams.splitter = constants.oldSplitter; - } - return services.metadataStoreMPObject(MPUbucket.getName(), - cipherBundle, metadataStoreParams, - log, (err, mpuMD) => { - if (err) { - log.trace('error storing multipart object', { - error: err, - }); - monitoring.promMetrics('PUT', bucketName, err.code, - 'initiateMultipartUpload'); - return callback(err, null, corsHeaders); - } - log.addDefaultFields({ uploadId }); - log.trace('successfully initiated mpu'); - pushMetric('initiateMultipartUpload', log, { - authInfo, - bucket: bucketName, - keys: [objectKey], - location: locConstraint, + services.getMPUBucket(destinationBucket, bucketName, log, (err, MPUbucket) => { + if (err) { + log.trace('error getting MPUbucket', { + error: err, + }); + return callback(err); + } + // BACKWARD: Remove to remove the old splitter + if (MPUbucket.getMdBucketModelVersion() < 2) { + metadataStoreParams.splitter = constants.oldSplitter; + } + return services.metadataStoreMPObject( + MPUbucket.getName(), + cipherBundle, + metadataStoreParams, + log, + (err, mpuMD) => { + if (err) { + log.trace('error storing multipart object', { + error: err, }); + monitoring.promMetrics('PUT', bucketName, err.code, 'initiateMultipartUpload'); + return callback(err, null, corsHeaders); + } + log.addDefaultFields({ uploadId }); + log.trace('successfully initiated mpu'); + pushMetric('initiateMultipartUpload', log, { + authInfo, + bucket: bucketName, + keys: [objectKey], + location: locConstraint, + }); - // TODO: rename corsHeaders to headers - setExpirationHeaders(corsHeaders, { - lifecycleConfig: destinationBucket.getLifecycleConfiguration(), - mpuParams: { - key: mpuMD.key, - date: mpuMD.initiated, - }, - }); + // TODO: rename corsHeaders to headers + setExpirationHeaders(corsHeaders, { + lifecycleConfig: destinationBucket.getLifecycleConfiguration(), + mpuParams: { + key: mpuMD.key, + date: mpuMD.initiated, + }, + }); - setSSEHeaders(corsHeaders, - mpuMD['x-amz-server-side-encryption'], - mpuMD['x-amz-server-side-encryption-aws-kms-key-id']); + setSSEHeaders( + corsHeaders, + mpuMD['x-amz-server-side-encryption'], + mpuMD['x-amz-server-side-encryption-aws-kms-key-id'], + ); - monitoring.promMetrics('PUT', bucketName, '200', - 'initiateMultipartUpload'); - return callback(null, xml, corsHeaders); - }); - }); + monitoring.promMetrics('PUT', bucketName, '200', 'initiateMultipartUpload'); + return callback(null, xml, corsHeaders); + }, + ); + }); } function _storetheMPObject(destinationBucket, corsHeaders, serverSideEncryption) { @@ -224,8 +220,7 @@ function initiateMultipartUpload(authInfo, request, log, callback) { masterKeyId: configuredMasterKeyId || masterKeyId, }; } - const backendInfoObj = locationConstraintCheck(request, null, - destinationBucket, log); + const backendInfoObj = locationConstraintCheck(request, null, destinationBucket, log); if (backendInfoObj.err) { return process.nextTick(() => { callback(backendInfoObj.err); @@ -236,21 +231,17 @@ function initiateMultipartUpload(authInfo, request, log, callback) { metadataStoreParams.dataStoreName = locConstraint; if (request.headers) { - const objectLockValError = - validateHeaders(destinationBucket, request.headers, log); + const objectLockValError = validateHeaders(destinationBucket, request.headers, log); if (objectLockValError) { return callback(objectLockValError); } } const defaultRetention = destinationBucket.getObjectLockConfiguration(); - const finalObjectLockInfo = - compareObjectLockInformation(request.headers, defaultRetention); + const finalObjectLockInfo = compareObjectLockInformation(request.headers, defaultRetention); if (finalObjectLockInfo.retentionInfo) { - metadataStoreParams.retentionMode = - finalObjectLockInfo.retentionInfo.mode; - metadataStoreParams.retentionDate = - finalObjectLockInfo.retentionInfo.date; + metadataStoreParams.retentionMode = finalObjectLockInfo.retentionInfo.mode; + metadataStoreParams.retentionDate = finalObjectLockInfo.retentionInfo.date; } if (finalObjectLockInfo.legalHold) { metadataStoreParams.legalHold = finalObjectLockInfo.legalHold; @@ -268,9 +259,11 @@ function initiateMultipartUpload(authInfo, request, log, callback) { const putVersionId = request.headers['x-scal-s3-version-id']; const isPutVersion = putVersionId || putVersionId === ''; - if (isPutVersion && + if ( + isPutVersion && locConstraint === destinationBucket.getLocationConstraint() && - destinationBucket.isIngestionBucket()) { + destinationBucket.isIngestionBucket() + ) { // When restoring to OOB bucket, we cannot force the versionId of the object written to the // backend, and it is thus not match the versionId of the ingested object. Thus we add extra // user metadata to allow OOB to allow ingestion processor to "match" the (new) restored @@ -278,18 +271,15 @@ function initiateMultipartUpload(authInfo, request, log, callback) { mpuInfo.metaHeaders['x-amz-meta-scal-version-id'] = putVersionId; } - return data.initiateMPU(mpuInfo, websiteRedirectHeader, log, - (err, dataBackendResObj, isVersionedObj) => { + return data.initiateMPU(mpuInfo, websiteRedirectHeader, log, (err, dataBackendResObj, isVersionedObj) => { // will return as true and a custom error if external backend does // not support versioned objects if (isVersionedObj) { - monitoring.promMetrics('PUT', bucketName, 501, - 'initiateMultipartUpload'); + monitoring.promMetrics('PUT', bucketName, 501, 'initiateMultipartUpload'); return callback(err); } if (err) { - monitoring.promMetrics('PUT', bucketName, err.code, - 'initiateMultipartUpload'); + monitoring.promMetrics('PUT', bucketName, err.code, 'initiateMultipartUpload'); return callback(err); } // if mpu not handled externally, dataBackendResObj will be null @@ -299,64 +289,71 @@ function initiateMultipartUpload(authInfo, request, log, callback) { // Generate uniqueID without dashes so routing not messed up uploadId = uuidv4().replace(/-/g, ''); } - return _getMPUBucket(destinationBucket, log, corsHeaders, - uploadId, cipherBundle, locConstraint, callback); + return _getMPUBucket(destinationBucket, log, corsHeaders, uploadId, cipherBundle, locConstraint, callback); }); } - async.waterfall([ - next => standardMetadataValidateBucketAndObj(metadataValParams, request.actionImplicitDenies, log, - (error, destinationBucket, destObjMD) => - updateEncryption(error, destinationBucket, destObjMD, objectKey, log, { skipObject: true }, - (error, destinationBucket) => { - const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, destinationBucket); - if (error) { - log.debug('error processing request', { - error, - method: 'metadataValidateBucketAndObj', + async.waterfall( + [ + next => + standardMetadataValidateBucketAndObj( + metadataValParams, + request.actionImplicitDenies, + log, + (error, destinationBucket, destObjMD) => + updateEncryption( + error, + destinationBucket, + destObjMD, + objectKey, + log, + { skipObject: true }, + (error, destinationBucket) => { + const corsHeaders = collectCorsHeaders( + request.headers.origin, + request.method, + destinationBucket, + ); + if (error) { + log.debug('error processing request', { + error, + method: 'metadataValidateBucketAndObj', + }); + monitoring.promMetrics('PUT', bucketName, error.code, 'initiateMultipartUpload'); + return next(error, corsHeaders); + } + return next(null, corsHeaders, destinationBucket); + }, + ), + ), + (corsHeaders, destinationBucket, next) => { + if (destinationBucket.hasDeletedFlag() && accountCanonicalID !== destinationBucket.getOwner()) { + log.trace('deleted flag on bucket and request from non-owner account'); + monitoring.promMetrics('PUT', bucketName, 404, 'initiateMultipartUpload'); + return next(errors.NoSuchBucket, corsHeaders); + } + if (destinationBucket.hasTransientFlag() || destinationBucket.hasDeletedFlag()) { + log.trace('transient or deleted flag so cleaning up bucket'); + return cleanUpBucket(destinationBucket, accountCanonicalID, log, error => { + if (error) { + log.debug('error cleaning up bucket with flag', { + error, + transientFlag: destinationBucket.hasTransientFlag(), + deletedFlag: destinationBucket.hasDeletedFlag(), + }); + // To avoid confusing user with error + // from cleaning up + // bucket return InternalError + monitoring.promMetrics('PUT', bucketName, 500, 'initiateMultipartUpload'); + return next(errors.InternalError, corsHeaders); + } + return next(null, corsHeaders, destinationBucket); }); - monitoring.promMetrics('PUT', bucketName, error.code, 'initiateMultipartUpload'); - return next(error, corsHeaders); } return next(null, corsHeaders, destinationBucket); - })), - (corsHeaders, destinationBucket, next) => { - if (destinationBucket.hasDeletedFlag() && accountCanonicalID !== destinationBucket.getOwner()) { - log.trace('deleted flag on bucket and request from non-owner account'); - monitoring.promMetrics('PUT', bucketName, 404, 'initiateMultipartUpload'); - return next(errors.NoSuchBucket, corsHeaders); - } - if (destinationBucket.hasTransientFlag() || destinationBucket.hasDeletedFlag()) { - log.trace('transient or deleted flag so cleaning up bucket'); - return cleanUpBucket( - destinationBucket, - accountCanonicalID, - log, - error => { - if (error) { - log.debug('error cleaning up bucket with flag', - { - error, - transientFlag: destinationBucket.hasTransientFlag(), - deletedFlag: destinationBucket.hasDeletedFlag(), - }); - // To avoid confusing user with error - // from cleaning up - // bucket return InternalError - monitoring.promMetrics('PUT', bucketName, 500, 'initiateMultipartUpload'); - return next(errors.InternalError, corsHeaders); - } - return next(null, corsHeaders, destinationBucket); - }); - } - return next(null, corsHeaders, destinationBucket); - }, - (corsHeaders, destinationBucket, next) => - getObjectSSEConfiguration( - request.headers, - destinationBucket, - log, - (error, objectSSEConfig) => { + }, + (corsHeaders, destinationBucket, next) => + getObjectSSEConfiguration(request.headers, destinationBucket, log, (error, objectSSEConfig) => { if (error) { log.error('error fetching server-side encryption config', { error, @@ -365,23 +362,23 @@ function initiateMultipartUpload(authInfo, request, log, callback) { return next(error, corsHeaders); } return next(null, corsHeaders, destinationBucket, objectSSEConfig); + }), + // If SSE configured, test kms key encryption access, but ignore cipher bundle + (corsHeaders, destinationBucket, objectSSEConfig, next) => { + if (objectSSEConfig) { + return kms.createCipherBundle(objectSSEConfig, log, err => + next(err, corsHeaders, destinationBucket, objectSSEConfig), + ); } - ), - // If SSE configured, test kms key encryption access, but ignore cipher bundle - (corsHeaders, destinationBucket, objectSSEConfig, next) => { - if (objectSSEConfig) { - return kms.createCipherBundle(objectSSEConfig, log, - err => next(err, corsHeaders, destinationBucket, objectSSEConfig)); - } - return next(null, corsHeaders, destinationBucket, objectSSEConfig); - }, - ], + return next(null, corsHeaders, destinationBucket, objectSSEConfig); + }, + ], (error, corsHeaders, destinationBucket, objectSSEConfig) => { if (error) { return callback(error, null, corsHeaders); } return _storetheMPObject(destinationBucket, corsHeaders, objectSSEConfig); - } + }, ); return undefined; } diff --git a/lib/api/listMultipartUploads.js b/lib/api/listMultipartUploads.js index 71e428669c..cc54ee8e19 100644 --- a/lib/api/listMultipartUploads.js +++ b/lib/api/listMultipartUploads.js @@ -101,72 +101,71 @@ function listMultipartUploads(authInfo, request, log, callback) { request, }; - async.waterfall([ - function waterfall1(next) { - // Check final destination bucket for authorization rather - // than multipart upload bucket - standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, - (err, bucket) => next(err, bucket)); - }, - function getMPUBucket(bucket, next) { - services.getMPUBucket(bucket, bucketName, log, - (err, mpuBucket) => next(err, bucket, mpuBucket)); - }, - function waterfall2(bucket, mpuBucket, next) { - let splitter = constants.splitter; - // BACKWARD: Remove to remove the old splitter - if (mpuBucket.getMdBucketModelVersion() < 2) { - splitter = constants.oldSplitter; - } - let maxUploads = query['max-uploads'] !== undefined ? - Number.parseInt(query['max-uploads'], 10) : 1000; - if (maxUploads < 0) { - monitoring.promMetrics('GET', bucketName, 400, - 'listMultipartUploads'); - return callback(errors.InvalidArgument, bucket); - } - if (maxUploads > constants.listingHardLimit) { - maxUploads = constants.listingHardLimit; + async.waterfall( + [ + function waterfall1(next) { + // Check final destination bucket for authorization rather + // than multipart upload bucket + standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => + next(err, bucket), + ); + }, + function getMPUBucket(bucket, next) { + services.getMPUBucket(bucket, bucketName, log, (err, mpuBucket) => next(err, bucket, mpuBucket)); + }, + function waterfall2(bucket, mpuBucket, next) { + let splitter = constants.splitter; + // BACKWARD: Remove to remove the old splitter + if (mpuBucket.getMdBucketModelVersion() < 2) { + splitter = constants.oldSplitter; + } + let maxUploads = query['max-uploads'] !== undefined ? Number.parseInt(query['max-uploads'], 10) : 1000; + if (maxUploads < 0) { + monitoring.promMetrics('GET', bucketName, 400, 'listMultipartUploads'); + return callback(errors.InvalidArgument, bucket); + } + if (maxUploads > constants.listingHardLimit) { + maxUploads = constants.listingHardLimit; + } + const listingParams = { + delimiter: query.delimiter, + keyMarker: query['key-marker'], + uploadIdMarker: query['upload-id-marker'], + maxKeys: maxUploads, + prefix: `overview${splitter}${prefix}`, + queryPrefixLength: prefix.length, + listingType: 'MPU', + splitter, + }; + services.getMultipartUploadListing(mpuBucketName, listingParams, log, (err, list) => + next(err, bucket, list), + ); + return undefined; + }, + ], + (err, bucket, list) => { + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + if (err) { + monitoring.promMetrics('GET', bucketName, err.code, 'listMultipartUploads'); + return callback(err, null, corsHeaders); } - const listingParams = { - delimiter: query.delimiter, + const xmlParams = { + bucketName, + encoding, + list, + prefix: query.prefix, keyMarker: query['key-marker'], uploadIdMarker: query['upload-id-marker'], - maxKeys: maxUploads, - prefix: `overview${splitter}${prefix}`, - queryPrefixLength: prefix.length, - listingType: 'MPU', - splitter, }; - services.getMultipartUploadListing(mpuBucketName, listingParams, - log, (err, list) => next(err, bucket, list)); - return undefined; + const xml = convertToXml('listMultipartUploads', xmlParams); + pushMetric('listMultipartUploads', log, { + authInfo, + bucket: bucketName, + }); + monitoring.promMetrics('GET', bucketName, '200', 'listMultipartUploads'); + return callback(null, xml, corsHeaders); }, - ], (err, bucket, list) => { - const corsHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); - if (err) { - monitoring.promMetrics('GET', bucketName, err.code, - 'listMultipartUploads'); - return callback(err, null, corsHeaders); - } - const xmlParams = { - bucketName, - encoding, - list, - prefix: query.prefix, - keyMarker: query['key-marker'], - uploadIdMarker: query['upload-id-marker'], - }; - const xml = convertToXml('listMultipartUploads', xmlParams); - pushMetric('listMultipartUploads', log, { - authInfo, - bucket: bucketName, - }); - monitoring.promMetrics( - 'GET', bucketName, '200', 'listMultipartUploads'); - return callback(null, xml, corsHeaders); - }); + ); } module.exports = listMultipartUploads; diff --git a/lib/api/listParts.js b/lib/api/listParts.js index 9ceec93e6e..c9b8e230ee 100644 --- a/lib/api/listParts.js +++ b/lib/api/listParts.js @@ -5,8 +5,7 @@ const { errors, s3middleware } = require('arsenal'); const constants = require('../../constants'); const collectCorsHeaders = require('../utilities/collectCorsHeaders'); -const locationConstraintCheck = - require('./apiUtils/object/locationConstraintCheck'); +const locationConstraintCheck = require('./apiUtils/object/locationConstraintCheck'); const services = require('../services'); const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUtils'); const escapeForXml = s3middleware.escapeForXml; @@ -57,8 +56,7 @@ function buildXML(xmlParams, xml, encodingFn) { xmlParams.forEach(param => { if (param.value !== undefined) { xml.push(`<${param.tag}>${encodingFn(param.value)}`); - } else if (param.tag !== 'NextPartNumberMarker' && - param.tag !== 'PartNumberMarker') { + } else if (param.tag !== 'NextPartNumberMarker' && param.tag !== 'PartNumberMarker') { xml.push(`<${param.tag}/>`); } }); @@ -79,19 +77,19 @@ function listParts(authInfo, request, log, callback) { const objectKey = request.objectKey; const uploadId = request.query.uploadId; const encoding = request.query['encoding-type']; - let maxParts = Number.parseInt(request.query['max-parts'], 10) ? - Number.parseInt(request.query['max-parts'], 10) : 1000; + let maxParts = Number.parseInt(request.query['max-parts'], 10) + ? Number.parseInt(request.query['max-parts'], 10) + : 1000; if (maxParts < 0) { - monitoring.promMetrics('GET', bucketName, 400, - 'listMultipartUploadParts'); + monitoring.promMetrics('GET', bucketName, 400, 'listMultipartUploadParts'); return callback(errors.InvalidArgument); } if (maxParts > constants.listingHardLimit) { maxParts = constants.listingHardLimit; } - const partNumberMarker = - Number.parseInt(request.query['part-number-marker'], 10) ? - Number.parseInt(request.query['part-number-marker'], 10) : 0; + const partNumberMarker = Number.parseInt(request.query['part-number-marker'], 10) + ? Number.parseInt(request.query['part-number-marker'], 10) + : 0; const metadataValMPUparams = { authInfo, bucketName, @@ -112,192 +110,202 @@ function listParts(authInfo, request, log, callback) { let splitter = constants.splitter; const responseHeaders = {}; - async.waterfall([ - function checkDestBucketVal(next) { - standardMetadataValidateBucketAndObj(metadataValParams, request.actionImplicitDenies, log, - (err, destinationBucket) => { - if (err) { - return next(err, destinationBucket, null); - } - if (destinationBucket.policies) { - // TODO: Check bucket policies to see if user is granted - // permission or forbidden permission to take - // given action. - // If permitted, add 'bucketPolicyGoAhead' - // attribute to params for validating at MPU level. - // This is GH Issue#76 - metadataValMPUparams.requestType = - 'bucketPolicyGoAhead'; - } - return next(null, destinationBucket); - }); - }, - function waterfall2(destBucket, next) { - metadataValMPUparams.log = log; - services.metadataValidateMultipart(metadataValMPUparams, - (err, mpuBucket, mpuOverviewObj) => { + async.waterfall( + [ + function checkDestBucketVal(next) { + standardMetadataValidateBucketAndObj( + metadataValParams, + request.actionImplicitDenies, + log, + (err, destinationBucket) => { + if (err) { + return next(err, destinationBucket, null); + } + if (destinationBucket.policies) { + // TODO: Check bucket policies to see if user is granted + // permission or forbidden permission to take + // given action. + // If permitted, add 'bucketPolicyGoAhead' + // attribute to params for validating at MPU level. + // This is GH Issue#76 + metadataValMPUparams.requestType = 'bucketPolicyGoAhead'; + } + return next(null, destinationBucket); + }, + ); + }, + function waterfall2(destBucket, next) { + metadataValMPUparams.log = log; + services.metadataValidateMultipart(metadataValMPUparams, (err, mpuBucket, mpuOverviewObj) => { if (err) { return next(err, destBucket, null); } return next(null, destBucket, mpuBucket, mpuOverviewObj); }); - }, - function waterfall3(destBucket, mpuBucket, mpuOverviewObj, next) { - const mpuInfo = { - objectKey, - uploadId, - bucketName, - partNumberMarker, - maxParts, - mpuOverviewObj, - destBucket, - }; - const originalIdentityImpDenies = request.actionImplicitDenies; - // eslint-disable-next-line no-param-reassign - delete request.actionImplicitDenies; - return data.listParts(mpuInfo, request, locationConstraintCheck, - log, (err, backendPartList) => { + }, + function waterfall3(destBucket, mpuBucket, mpuOverviewObj, next) { + const mpuInfo = { + objectKey, + uploadId, + bucketName, + partNumberMarker, + maxParts, + mpuOverviewObj, + destBucket, + }; + const originalIdentityImpDenies = request.actionImplicitDenies; // eslint-disable-next-line no-param-reassign - request.actionImplicitDenies = originalIdentityImpDenies; - if (err) { - return next(err, destBucket); - } - // if external backend doesn't handle mpu, backendPartList - // will be null - return next(null, destBucket, mpuBucket, mpuOverviewObj, - backendPartList); - }); - }, - function waterfall4(destBucket, mpuBucket, mpuOverviewObj, - backendPartList, next) { - // if parts were returned from cloud backend, they were not - // stored in Scality S3 metadata, so this step can be skipped - if (backendPartList) { - return next(null, destBucket, mpuBucket, backendPartList, - mpuOverviewObj); - } - // BACKWARD: Remove to remove the old splitter - if (mpuBucket.getMdBucketModelVersion() < 2) { - splitter = constants.oldSplitter; - } - const getPartsParams = { - uploadId, - mpuBucketName: mpuBucket.getName(), - maxParts, - partNumberMarker, - log, - splitter, - }; - return services.getSomeMPUparts(getPartsParams, - (err, storedParts) => { - if (err) { - return next(err, destBucket, null); + delete request.actionImplicitDenies; + return data.listParts(mpuInfo, request, locationConstraintCheck, log, (err, backendPartList) => { + // eslint-disable-next-line no-param-reassign + request.actionImplicitDenies = originalIdentityImpDenies; + if (err) { + return next(err, destBucket); + } + // if external backend doesn't handle mpu, backendPartList + // will be null + return next(null, destBucket, mpuBucket, mpuOverviewObj, backendPartList); + }); + }, + function waterfall4(destBucket, mpuBucket, mpuOverviewObj, backendPartList, next) { + // if parts were returned from cloud backend, they were not + // stored in Scality S3 metadata, so this step can be skipped + if (backendPartList) { + return next(null, destBucket, mpuBucket, backendPartList, mpuOverviewObj); } - return next(null, destBucket, mpuBucket, storedParts, - mpuOverviewObj); - }); - }, function waterfall5(destBucket, mpuBucket, storedParts, - mpuOverviewObj, next) { - const encodingFn = encoding === 'url' - ? querystring.escape : escapeForXml; - const isTruncated = storedParts.IsTruncated; - const splitterLen = splitter.length; - const partListing = storedParts.Contents.map(item => { - // key form: - // - {uploadId} - // - {splitter} - // - {partNumber} - let partNumber; - if (item.key) { - const index = item.key.lastIndexOf(splitter); - partNumber = - parseInt(item.key.substring(index + splitterLen), 10); - } else { - // if partListing came from real AWS backend, - // item.partNumber is present instead of item.key - partNumber = item.partNumber; + // BACKWARD: Remove to remove the old splitter + if (mpuBucket.getMdBucketModelVersion() < 2) { + splitter = constants.oldSplitter; } - return { - partNumber, - lastModified: item.value.LastModified, - ETag: item.value.ETag, - size: item.value.Size, + const getPartsParams = { + uploadId, + mpuBucketName: mpuBucket.getName(), + maxParts, + partNumberMarker, + log, + splitter, }; - }); - const lastPartShown = partListing.length > 0 ? - partListing[partListing.length - 1].partNumber : undefined; + return services.getSomeMPUparts(getPartsParams, (err, storedParts) => { + if (err) { + return next(err, destBucket, null); + } + return next(null, destBucket, mpuBucket, storedParts, mpuOverviewObj); + }); + }, + function waterfall5(destBucket, mpuBucket, storedParts, mpuOverviewObj, next) { + const encodingFn = encoding === 'url' ? querystring.escape : escapeForXml; + const isTruncated = storedParts.IsTruncated; + const splitterLen = splitter.length; + const partListing = storedParts.Contents.map(item => { + // key form: + // - {uploadId} + // - {splitter} + // - {partNumber} + let partNumber; + if (item.key) { + const index = item.key.lastIndexOf(splitter); + partNumber = parseInt(item.key.substring(index + splitterLen), 10); + } else { + // if partListing came from real AWS backend, + // item.partNumber is present instead of item.key + partNumber = item.partNumber; + } + return { + partNumber, + lastModified: item.value.LastModified, + ETag: item.value.ETag, + size: item.value.Size, + }; + }); + const lastPartShown = + partListing.length > 0 ? partListing[partListing.length - 1].partNumber : undefined; - setExpirationHeaders(responseHeaders, { - lifecycleConfig: destBucket.getLifecycleConfiguration(), - mpuParams: { - key: mpuOverviewObj.key, - date: mpuOverviewObj.initiated, - }, - }); + setExpirationHeaders(responseHeaders, { + lifecycleConfig: destBucket.getLifecycleConfiguration(), + mpuParams: { + key: mpuOverviewObj.key, + date: mpuOverviewObj.initiated, + }, + }); - const xml = []; - xml.push( - '', - '' - ); - buildXML([ - { tag: 'Bucket', value: bucketName }, - { tag: 'Key', value: objectKey }, - { tag: 'UploadId', value: uploadId }, - ], xml, encodingFn); - xml.push(''); - buildXML([ - { tag: 'ID', value: mpuOverviewObj.initiatorID }, - { tag: 'DisplayName', - value: mpuOverviewObj.initiatorDisplayName }, - ], xml, encodingFn); - xml.push(''); - xml.push(''); - buildXML([ - { tag: 'ID', value: mpuOverviewObj.ownerID }, - { tag: 'DisplayName', value: mpuOverviewObj.ownerDisplayName }, - ], xml, encodingFn); - xml.push(''); - buildXML([ - { tag: 'StorageClass', value: mpuOverviewObj.storageClass }, - { tag: 'PartNumberMarker', value: partNumberMarker || - undefined }, - // print only if it's truncated - { tag: 'NextPartNumberMarker', value: isTruncated ? - parseInt(lastPartShown, 10) : undefined }, - { tag: 'MaxParts', value: maxParts }, - { tag: 'IsTruncated', value: isTruncated ? 'true' : 'false' }, - ], xml, encodingFn); + const xml = []; + xml.push( + '', + '', + ); + buildXML( + [ + { tag: 'Bucket', value: bucketName }, + { tag: 'Key', value: objectKey }, + { tag: 'UploadId', value: uploadId }, + ], + xml, + encodingFn, + ); + xml.push(''); + buildXML( + [ + { tag: 'ID', value: mpuOverviewObj.initiatorID }, + { tag: 'DisplayName', value: mpuOverviewObj.initiatorDisplayName }, + ], + xml, + encodingFn, + ); + xml.push(''); + xml.push(''); + buildXML( + [ + { tag: 'ID', value: mpuOverviewObj.ownerID }, + { tag: 'DisplayName', value: mpuOverviewObj.ownerDisplayName }, + ], + xml, + encodingFn, + ); + xml.push(''); + buildXML( + [ + { tag: 'StorageClass', value: mpuOverviewObj.storageClass }, + { tag: 'PartNumberMarker', value: partNumberMarker || undefined }, + // print only if it's truncated + { tag: 'NextPartNumberMarker', value: isTruncated ? parseInt(lastPartShown, 10) : undefined }, + { tag: 'MaxParts', value: maxParts }, + { tag: 'IsTruncated', value: isTruncated ? 'true' : 'false' }, + ], + xml, + encodingFn, + ); - partListing.forEach(part => { - xml.push(''); - buildXML([ - { tag: 'PartNumber', value: part.partNumber }, - { tag: 'LastModified', value: part.lastModified }, - { tag: 'ETag', value: `"${part.ETag}"` }, - { tag: 'Size', value: part.size }, - ], xml, encodingFn); - xml.push(''); - }); - xml.push(''); - pushMetric('listMultipartUploadParts', log, { - authInfo, - bucket: bucketName, - }); - monitoring.promMetrics( - 'GET', bucketName, '200', 'listMultipartUploadParts'); - next(null, destBucket, xml.join('')); - }, - ], (err, destinationBucket, xml) => { - const corsHeaders = collectCorsHeaders(request.headers.origin, - request.method, destinationBucket); - monitoring.promMetrics('GET', bucketName, 400, - 'listMultipartUploadParts'); - Object.assign(responseHeaders, corsHeaders); + partListing.forEach(part => { + xml.push(''); + buildXML( + [ + { tag: 'PartNumber', value: part.partNumber }, + { tag: 'LastModified', value: part.lastModified }, + { tag: 'ETag', value: `"${part.ETag}"` }, + { tag: 'Size', value: part.size }, + ], + xml, + encodingFn, + ); + xml.push(''); + }); + xml.push(''); + pushMetric('listMultipartUploadParts', log, { + authInfo, + bucket: bucketName, + }); + monitoring.promMetrics('GET', bucketName, '200', 'listMultipartUploadParts'); + next(null, destBucket, xml.join('')); + }, + ], + (err, destinationBucket, xml) => { + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, destinationBucket); + monitoring.promMetrics('GET', bucketName, 400, 'listMultipartUploadParts'); + Object.assign(responseHeaders, corsHeaders); - return callback(err, xml, responseHeaders); - }); + return callback(err, xml, responseHeaders); + }, + ); return undefined; } diff --git a/lib/api/metadataSearch.js b/lib/api/metadataSearch.js index ba4f355abc..382472eb08 100644 --- a/lib/api/metadataSearch.js +++ b/lib/api/metadataSearch.js @@ -8,13 +8,10 @@ const validateSearchParams = require('../api/apiUtils/bucket/validateSearch'); const parseWhere = require('../api/apiUtils/bucket/parseWhere'); const versionIdUtils = versioning.VersionID; const monitoring = require('../utilities/monitoringHandler'); -const { decryptToken } - = require('../api/apiUtils/object/continueToken'); +const { decryptToken } = require('../api/apiUtils/object/continueToken'); const { processVersions, processMasterVersions } = require('./bucketGet'); - -function handleResult(listParams, requestMaxKeys, encoding, authInfo, - bucketName, list, corsHeaders, log, callback) { +function handleResult(listParams, requestMaxKeys, encoding, authInfo, bucketName, list, corsHeaders, log, callback) { // eslint-disable-next-line no-param-reassign listParams.maxKeys = requestMaxKeys; // eslint-disable-next-line no-param-reassign @@ -47,22 +44,21 @@ function metadataSearch(authInfo, request, log, callback) { const bucketName = request.bucketName; const v2 = params['list-type']; if (v2 !== undefined && Number.parseInt(v2, 10) !== 2) { - return callback(errorInstances.InvalidArgument.customizeDescription('Invalid ' + - 'List Type specified in Request')); + return callback( + errorInstances.InvalidArgument.customizeDescription('Invalid ' + 'List Type specified in Request'), + ); } log.debug('processing request', { method: 'metadataSearch' }); const encoding = params['encoding-type']; if (encoding !== undefined && encoding !== 'url') { - monitoring.promMetrics( - 'GET', bucketName, 400, 'metadataSearch'); - return callback(errorInstances.InvalidArgument.customizeDescription('Invalid ' + - 'Encoding Method specified in Request')); + monitoring.promMetrics('GET', bucketName, 400, 'metadataSearch'); + return callback( + errorInstances.InvalidArgument.customizeDescription('Invalid ' + 'Encoding Method specified in Request'), + ); } - const requestMaxKeys = params['max-keys'] ? - Number.parseInt(params['max-keys'], 10) : 1000; + const requestMaxKeys = params['max-keys'] ? Number.parseInt(params['max-keys'], 10) : 1000; if (Number.isNaN(requestMaxKeys) || requestMaxKeys < 0) { - monitoring.promMetrics( - 'GET', bucketName, 400, 'metadataSearch'); + monitoring.promMetrics('GET', bucketName, 400, 'metadataSearch'); return callback(errors.InvalidArgument); } // AWS only returns 1000 keys even if max keys are greater. @@ -89,37 +85,34 @@ function metadataSearch(authInfo, request, log, callback) { log.debug(err.message, { stack: err.stack, }); - monitoring.promMetrics( - 'GET', bucketName, 400, 'metadataSearch'); - return callback(errorInstances.InvalidArgument - .customizeDescription('Invalid sql where clause ' + - 'sent as search query')); + monitoring.promMetrics('GET', bucketName, 400, 'metadataSearch'); + return callback( + errorInstances.InvalidArgument.customizeDescription('Invalid sql where clause ' + 'sent as search query'), + ); } if (v2) { listParams.v2 = true; listParams.startAfter = params['start-after']; - listParams.continuationToken = - decryptToken(params['continuation-token']); + listParams.continuationToken = decryptToken(params['continuation-token']); listParams.fetchOwner = params['fetch-owner'] === 'true'; } else { listParams.marker = params.marker; } standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => { - const corsHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); if (err) { log.debug('error processing request', { error: err }); - monitoring.promMetrics( - 'GET', bucketName, err.code, 'metadataSearch'); + monitoring.promMetrics('GET', bucketName, err.code, 'metadataSearch'); return callback(err, null, corsHeaders); } if (params.versions !== undefined) { listParams.listingType = 'DelimiterVersions'; delete listParams.marker; listParams.keyMarker = params['key-marker']; - listParams.versionIdMarker = params['version-id-marker'] ? - versionIdUtils.decode(params['version-id-marker']) : undefined; + listParams.versionIdMarker = params['version-id-marker'] + ? versionIdUtils.decode(params['version-id-marker']) + : undefined; } if (!requestMaxKeys) { const emptyList = { @@ -128,20 +121,36 @@ function metadataSearch(authInfo, request, log, callback) { Versions: [], IsTruncated: false, }; - return handleResult(listParams, requestMaxKeys, encoding, authInfo, - bucketName, emptyList, corsHeaders, log, callback); + return handleResult( + listParams, + requestMaxKeys, + encoding, + authInfo, + bucketName, + emptyList, + corsHeaders, + log, + callback, + ); } - return services.getObjectListing(bucketName, listParams, log, - (err, list) => { - if (err) { - log.debug('error processing request', { error: err }); - monitoring.promMetrics( - 'GET', bucketName, err.code, 'metadataSearch'); - return callback(err, null, corsHeaders); - } - return handleResult(listParams, requestMaxKeys, encoding, authInfo, - bucketName, list, corsHeaders, log, callback); - }); + return services.getObjectListing(bucketName, listParams, log, (err, list) => { + if (err) { + log.debug('error processing request', { error: err }); + monitoring.promMetrics('GET', bucketName, err.code, 'metadataSearch'); + return callback(err, null, corsHeaders); + } + return handleResult( + listParams, + requestMaxKeys, + encoding, + authInfo, + bucketName, + list, + corsHeaders, + log, + callback, + ); + }); }); return undefined; } diff --git a/lib/api/multipartDelete.js b/lib/api/multipartDelete.js index 48146774b7..bb43b86291 100644 --- a/lib/api/multipartDelete.js +++ b/lib/api/multipartDelete.js @@ -22,12 +22,15 @@ function multipartDelete(authInfo, request, log, callback) { const objectKey = request.objectKey; const uploadId = request.query.uploadId; - abortMultipartUpload(authInfo, bucketName, objectKey, uploadId, log, + abortMultipartUpload( + authInfo, + bucketName, + objectKey, + uploadId, + log, (err, destinationBucket, partSizeSum) => { - const corsHeaders = collectCorsHeaders(request.headers.origin, - request.method, destinationBucket); - const location = destinationBucket ? - destinationBucket.getLocationConstraint() : null; + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, destinationBucket); + const location = destinationBucket ? destinationBucket.getLocationConstraint() : null; if (err && !err?.is?.NoSuchUpload) { return callback(err, corsHeaders); } @@ -36,12 +39,10 @@ function multipartDelete(authInfo, request, log, callback) { method: 'multipartDelete', uploadId, }); - monitoring.promMetrics('DELETE', bucketName, 400, - 'abortMultipartUpload'); + monitoring.promMetrics('DELETE', bucketName, 400, 'abortMultipartUpload'); return callback(err, corsHeaders); } - monitoring.promMetrics('DELETE', bucketName, 400, - 'abortMultipartUpload'); + monitoring.promMetrics('DELETE', bucketName, 400, 'abortMultipartUpload'); if (!err) { pushMetric('abortMultipartUpload', log, { authInfo, @@ -61,7 +62,9 @@ function multipartDelete(authInfo, request, log, callback) { } } return callback(null, corsHeaders); - }, request); + }, + request, + ); } module.exports = multipartDelete; diff --git a/lib/api/objectCopy.js b/lib/api/objectCopy.js index 02b0a31d5b..7be00d440a 100644 --- a/lib/api/objectCopy.js +++ b/lib/api/objectCopy.js @@ -7,18 +7,15 @@ const validateHeaders = s3middleware.validateConditionalHeaders; const constants = require('../../constants'); const collectCorsHeaders = require('../utilities/collectCorsHeaders'); -const locationConstraintCheck - = require('./apiUtils/object/locationConstraintCheck'); -const { checkQueryVersionId, versioningPreprocessing, decodeVID } - = require('./apiUtils/object/versioning'); +const locationConstraintCheck = require('./apiUtils/object/locationConstraintCheck'); +const { checkQueryVersionId, versioningPreprocessing, decodeVID } = require('./apiUtils/object/versioning'); const getReplicationInfo = require('./apiUtils/object/getReplicationInfo'); const { data } = require('../data/wrapper'); const services = require('../services'); const { pushMetric } = require('../utapi/utilities'); const removeAWSChunked = require('./apiUtils/object/removeAWSChunked'); const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUtils'); -const validateWebsiteHeader = require('./apiUtils/object/websiteServing') - .validateWebsiteHeader; +const validateWebsiteHeader = require('./apiUtils/object/websiteServing').validateWebsiteHeader; const { config } = require('../Config'); const monitoring = require('../utilities/monitoringHandler'); const applyZenkoUserMD = require('./apiUtils/object/applyZenkoUserMD'); @@ -32,8 +29,8 @@ const { initializeInternalLogRequestQueue, queueInternalLogRequest } = require(' const versionIdUtils = versioning.VersionID; const locationHeader = constants.objectLocationConstraintHeader; const versioningNotImplBackends = constants.versioningNotImplBackends; -const externalVersioningErrorMessage = 'We do not currently support putting ' + -'a versioned object to a location-constraint of type AWS or Azure or GCP.'; +const externalVersioningErrorMessage = + 'We do not currently support putting ' + 'a versioned object to a location-constraint of type AWS or Azure or GCP.'; /** * Preps metadata to be saved (based on copy or replace request header) @@ -53,8 +50,18 @@ const externalVersioningErrorMessage = 'We do not currently support putting ' + * - sourceLocationConstraintName {string} - location type of the source * - OR error */ -function _prepMetadata(request, sourceObjMD, headers, sourceIsDestination, - authInfo, objectKey, sourceBucketMD, destBucketMD, sourceVersionId, log) { +function _prepMetadata( + request, + sourceObjMD, + headers, + sourceIsDestination, + authInfo, + objectKey, + sourceBucketMD, + destBucketMD, + sourceVersionId, + log, +) { let whichMetadata = headers['x-amz-metadata-directive']; // Default is COPY whichMetadata = whichMetadata === undefined ? 'COPY' : whichMetadata; @@ -65,45 +72,47 @@ function _prepMetadata(request, sourceObjMD, headers, sourceIsDestination, // Default is COPY whichTagging = whichTagging === undefined ? 'COPY' : whichTagging; if (whichTagging !== 'COPY' && whichTagging !== 'REPLACE') { - return { error: errorInstances.InvalidArgument - .customizeDescription('Unknown tagging directive') }; + return { error: errorInstances.InvalidArgument.customizeDescription('Unknown tagging directive') }; } const overrideMetadata = {}; if (headers['x-amz-server-side-encryption']) { - overrideMetadata['x-amz-server-side-encryption'] = - headers['x-amz-server-side-encryption']; + overrideMetadata['x-amz-server-side-encryption'] = headers['x-amz-server-side-encryption']; } - if (headers['x-amz-storage-class']) { // TODO: remove in CLDSRV-639 - overrideMetadata['x-amz-storage-class'] = - headers['x-amz-storage-class']; + if (headers['x-amz-storage-class']) { + // TODO: remove in CLDSRV-639 + overrideMetadata['x-amz-storage-class'] = headers['x-amz-storage-class']; } if (headers['x-amz-website-redirect-location']) { - overrideMetadata['x-amz-website-redirect-location'] = - headers['x-amz-website-redirect-location']; + overrideMetadata['x-amz-website-redirect-location'] = headers['x-amz-website-redirect-location']; } - const retentionHeaders = headers['x-amz-object-lock-mode'] - && headers['x-amz-object-lock-retain-until-date']; + const retentionHeaders = headers['x-amz-object-lock-mode'] && headers['x-amz-object-lock-retain-until-date']; const legalHoldHeader = headers['x-amz-object-lock-legal-hold']; - if ((retentionHeaders || legalHoldHeader) - && !destBucketMD.isObjectLockEnabled()) { - return { error: errorInstances.InvalidRequest.customizeDescription( - 'Bucket is missing ObjectLockConfiguration') }; + if ((retentionHeaders || legalHoldHeader) && !destBucketMD.isObjectLockEnabled()) { + return { + error: errorInstances.InvalidRequest.customizeDescription('Bucket is missing ObjectLockConfiguration'), + }; } // Cannot copy from same source and destination if no MD // changed and no source version id - if (sourceIsDestination && whichMetadata === 'COPY' && - Object.keys(overrideMetadata).length === 0 && !sourceVersionId) { - return { error: errorInstances.InvalidRequest.customizeDescription('This copy' + - ' request is illegal because it is trying to copy an ' + - 'object to itself without changing the object\'s metadata, ' + - 'storage class, website redirect location or encryption ' + - 'attributes.') }; + if ( + sourceIsDestination && + whichMetadata === 'COPY' && + Object.keys(overrideMetadata).length === 0 && + !sourceVersionId + ) { + return { + error: errorInstances.InvalidRequest.customizeDescription( + 'This copy' + + ' request is illegal because it is trying to copy an ' + + "object to itself without changing the object's metadata, " + + 'storage class, website redirect location or encryption ' + + 'attributes.', + ), + }; } // If COPY, pull all x-amz-meta keys/values from source object // Otherwise, pull all x-amz-meta keys/values from request headers - const userMetadata = whichMetadata === 'COPY' ? - getMetaHeaders(sourceObjMD) : - getMetaHeaders(headers); + const userMetadata = whichMetadata === 'COPY' ? getMetaHeaders(sourceObjMD) : getMetaHeaders(headers); if (userMetadata instanceof Error) { log.debug('user metadata validation failed', { error: userMetadata, @@ -117,28 +126,23 @@ function _prepMetadata(request, sourceObjMD, headers, sourceIsDestination, // If metadataDirective is: // - 'COPY' and source object has a location constraint in its metadata // we use the bucket destination location constraint - if (whichMetadata === 'COPY' - && userMetadata[locationHeader] - && destBucketMD.getLocationConstraint()) { + if (whichMetadata === 'COPY' && userMetadata[locationHeader] && destBucketMD.getLocationConstraint()) { userMetadata[locationHeader] = destBucketMD.getLocationConstraint(); } - const backendInfoObjSource = locationConstraintCheck(request, - sourceObjMD, sourceBucketMD, log); + const backendInfoObjSource = locationConstraintCheck(request, sourceObjMD, sourceBucketMD, log); if (backendInfoObjSource.err) { return { error: backendInfoObjSource.err }; } const sourceLocationConstraintName = backendInfoObjSource.controllingLC; - const backendInfoObjDest = locationConstraintCheck(request, - userMetadata, destBucketMD, log); + const backendInfoObjDest = locationConstraintCheck(request, userMetadata, destBucketMD, log); if (backendInfoObjDest.err) { return { error: backendInfoObjDest.err }; } const destLocationConstraintName = backendInfoObjDest.controllingLC; // If location constraint header is not included, locations match - const locationMatch = - sourceLocationConstraintName === destLocationConstraintName; + const locationMatch = sourceLocationConstraintName === destLocationConstraintName; // If tagging directive is REPLACE but you don't specify any // tags in the request, the destination object will @@ -155,8 +159,7 @@ function _prepMetadata(request, sourceObjMD, headers, sourceIsDestination, // If COPY, pull the necessary headers from source object // Otherwise, pull them from request headers - const headersToStoreSource = whichMetadata === 'COPY' ? - sourceObjMD : headers; + const headersToStoreSource = whichMetadata === 'COPY' ? sourceObjMD : headers; const storeMetadataParams = { objectKey, @@ -169,16 +172,14 @@ function _prepMetadata(request, sourceObjMD, headers, sourceIsDestination, contentMD5: sourceObjMD['content-md5'], cacheControl: headersToStoreSource['cache-control'], contentDisposition: headersToStoreSource['content-disposition'], - contentEncoding: - removeAWSChunked(headersToStoreSource['content-encoding']), + contentEncoding: removeAWSChunked(headersToStoreSource['content-encoding']), dataStoreName: destLocationConstraintName, expires: headersToStoreSource.expires, overrideMetadata, lastModifiedDate: new Date().toJSON(), tagging, taggingCopy, - replicationInfo: getReplicationInfo(config, - objectKey, destBucketMD, false, sourceObjMD['content-length']), + replicationInfo: getReplicationInfo(config, objectKey, destBucketMD, false, sourceObjMD['content-length']), locationMatch, originOp: 's3:ObjectCreated:Copy', }; @@ -198,8 +199,7 @@ function _prepMetadata(request, sourceObjMD, headers, sourceIsDestination, storeMetadataParams.bucketOwnerId = destBucketMD.getOwner(); } - return { storeMetadataParams, sourceLocationConstraintName, - backendInfoDest: backendInfoObjDest.backendInfo }; + return { storeMetadataParams, sourceLocationConstraintName, backendInfoDest: backendInfoObjDest.backendInfo }; } /** @@ -215,8 +215,7 @@ function _prepMetadata(request, sourceObjMD, headers, sourceIsDestination, * @param {function} callback - final callback to call with the result * @return {undefined} */ -function objectCopy(authInfo, request, sourceBucket, - sourceObject, sourceVersionId, log, callback) { +function objectCopy(authInfo, request, sourceBucket, sourceObject, sourceVersionId, log, callback) { log.debug('processing request', { method: 'objectCopy' }); const destBucketName = request.bucketName; const destObjectKey = request.objectKey; @@ -226,8 +225,7 @@ function objectCopy(authInfo, request, sourceBucket, return callback(keyLengthError); } - const sourceIsDestination = - destBucketName === sourceBucket && destObjectKey === sourceObject; + const sourceIsDestination = destBucketName === sourceBucket && destObjectKey === sourceObject; const valGetParams = { authInfo, bucketName: sourceBucket, @@ -259,432 +257,563 @@ function objectCopy(authInfo, request, sourceBucket, namespace: request.namespace, objectKey: destObjectKey, }; - const websiteRedirectHeader = - request.headers['x-amz-website-redirect-location']; + const websiteRedirectHeader = request.headers['x-amz-website-redirect-location']; const responseHeaders = {}; - if (request.headers['x-amz-storage-class'] && - !constants.validStorageClasses.includes(request.headers['x-amz-storage-class'])) { + if ( + request.headers['x-amz-storage-class'] && + !constants.validStorageClasses.includes(request.headers['x-amz-storage-class']) + ) { log.trace('invalid storage-class header'); - monitoring.promMetrics('PUT', destBucketName, - errorInstances.InvalidStorageClass.code, 'copyObject'); + monitoring.promMetrics('PUT', destBucketName, errorInstances.InvalidStorageClass.code, 'copyObject'); return callback(errors.InvalidStorageClass); } if (!validateWebsiteHeader(websiteRedirectHeader)) { const err = errors.InvalidRedirectLocation; - log.debug('invalid x-amz-website-redirect-location' + - `value ${websiteRedirectHeader}`, { error: err }); - monitoring.promMetrics( - 'PUT', destBucketName, err.code, 'copyObject'); + log.debug('invalid x-amz-website-redirect-location' + `value ${websiteRedirectHeader}`, { error: err }); + monitoring.promMetrics('PUT', destBucketName, err.code, 'copyObject'); return callback(err); } const queryContainsVersionId = checkQueryVersionId(request.query); if (queryContainsVersionId instanceof Error) { return callback(queryContainsVersionId); } - return async.waterfall([ - function checkDestAuth(next) { - return standardMetadataValidateBucketAndObj(valPutParams, request.actionImplicitDenies, log, - (err, destBucketMD, destObjMD) => - updateEncryption(err, destBucketMD, destObjMD, destObjectKey, log, { skipObject: true }, - (err, destBucketMD, destObjMD) => { - if (err) { - log.debug('error validating put part of request', - { error: err }); - return next(err, destBucketMD); - } - const flag = destBucketMD.hasDeletedFlag() - || destBucketMD.hasTransientFlag(); - if (flag) { - log.trace('deleted flag or transient flag ' + - 'on destination bucket', { flag }); - return next(errors.NoSuchBucket); - } - return next(null, destBucketMD, destObjMD); - })); - }, - function checkSourceAuthorization(destBucketMD, destObjMD, next) { - return standardMetadataValidateBucketAndObj({ - ...valGetParams, - destObjMD, - serverAccessLogOptions: { copySource: true }, - }, request.actionImplicitDenies, log, - (err, sourceBucketMD, sourceObjMD) => { - if (err) { - log.debug('error validating get part of request', - { error: err }); - // eslint-disable-next-line no-param-reassign - request.sourceServerAccessLog && (request.sourceServerAccessLog.error = err); - return next(err, null, destBucketMD); - } - if (!sourceObjMD) { - const err = sourceVersionId ? errors.NoSuchVersion : - errors.NoSuchKey; - log.debug('no source object', { sourceObject }); - // eslint-disable-next-line no-param-reassign - request.sourceServerAccessLog && (request.sourceServerAccessLog.error = err); - return next(err, null, destBucketMD); - } - // check if object data is in a cold storage - const coldErr = verifyColdObjectAvailable(sourceObjMD); - if (coldErr) { - // eslint-disable-next-line no-param-reassign - request.sourceServerAccessLog && (request.sourceServerAccessLog.error = coldErr); - return next(coldErr, null); - } - if (sourceObjMD.isDeleteMarker) { - log.debug('delete marker on source object', - { sourceObject }); - let err; - if (sourceVersionId) { - err = errorInstances.InvalidRequest - .customizeDescription('The source of a copy ' + - 'request may not specifically refer to a delete' + - 'marker by version id.'); - } else { - // if user specifies a key in a versioned source bucket - // without specifying a version, and the object has - // a delete marker, return NoSuchKey - err = errors.NoSuchKey; + return async.waterfall( + [ + function checkDestAuth(next) { + return standardMetadataValidateBucketAndObj( + valPutParams, + request.actionImplicitDenies, + log, + (err, destBucketMD, destObjMD) => + updateEncryption( + err, + destBucketMD, + destObjMD, + destObjectKey, + log, + { skipObject: true }, + (err, destBucketMD, destObjMD) => { + if (err) { + log.debug('error validating put part of request', { error: err }); + return next(err, destBucketMD); + } + const flag = destBucketMD.hasDeletedFlag() || destBucketMD.hasTransientFlag(); + if (flag) { + log.trace('deleted flag or transient flag ' + 'on destination bucket', { flag }); + return next(errors.NoSuchBucket); + } + return next(null, destBucketMD, destObjMD); + }, + ), + ); + }, + function checkSourceAuthorization(destBucketMD, destObjMD, next) { + return standardMetadataValidateBucketAndObj( + { + ...valGetParams, + destObjMD, + serverAccessLogOptions: { copySource: true }, + }, + request.actionImplicitDenies, + log, + (err, sourceBucketMD, sourceObjMD) => { + if (err) { + log.debug('error validating get part of request', { error: err }); + // eslint-disable-next-line no-param-reassign + request.sourceServerAccessLog && (request.sourceServerAccessLog.error = err); + return next(err, null, destBucketMD); } - // eslint-disable-next-line no-param-reassign - request.sourceServerAccessLog && (request.sourceServerAccessLog.error = err); - return next(err, destBucketMD); - } - const headerValResult = - validateHeaders(request.headers, - sourceObjMD['last-modified'], - sourceObjMD['content-md5']); - if (headerValResult.error) { - request.sourceServerAccessLog + if (!sourceObjMD) { + const err = sourceVersionId ? errors.NoSuchVersion : errors.NoSuchKey; + log.debug('no source object', { sourceObject }); // eslint-disable-next-line no-param-reassign - && (request.sourceServerAccessLog.error = errors.PreconditionFailed); - return next(errors.PreconditionFailed, destBucketMD); - } - const { storeMetadataParams, error: metadataError, - sourceLocationConstraintName, backendInfoDest } = - _prepMetadata(request, sourceObjMD, request.headers, - sourceIsDestination, authInfo, destObjectKey, - sourceBucketMD, destBucketMD, sourceVersionId, log); - if (metadataError) { - // eslint-disable-next-line no-param-reassign - request.sourceServerAccessLog && (request.sourceServerAccessLog.error = metadataError); - return next(metadataError, destBucketMD); - } - if (storeMetadataParams.metaHeaders) { - dataStoreContext.metaHeaders = - storeMetadataParams.metaHeaders; - } + request.sourceServerAccessLog && (request.sourceServerAccessLog.error = err); + return next(err, null, destBucketMD); + } + // check if object data is in a cold storage + const coldErr = verifyColdObjectAvailable(sourceObjMD); + if (coldErr) { + // eslint-disable-next-line no-param-reassign + request.sourceServerAccessLog && (request.sourceServerAccessLog.error = coldErr); + return next(coldErr, null); + } + if (sourceObjMD.isDeleteMarker) { + log.debug('delete marker on source object', { sourceObject }); + let err; + if (sourceVersionId) { + err = errorInstances.InvalidRequest.customizeDescription( + 'The source of a copy ' + + 'request may not specifically refer to a delete' + + 'marker by version id.', + ); + } else { + // if user specifies a key in a versioned source bucket + // without specifying a version, and the object has + // a delete marker, return NoSuchKey + err = errors.NoSuchKey; + } + // eslint-disable-next-line no-param-reassign + request.sourceServerAccessLog && (request.sourceServerAccessLog.error = err); + return next(err, destBucketMD); + } + const headerValResult = validateHeaders( + request.headers, + sourceObjMD['last-modified'], + sourceObjMD['content-md5'], + ); + if (headerValResult.error) { + request.sourceServerAccessLog && + // eslint-disable-next-line no-param-reassign + (request.sourceServerAccessLog.error = errors.PreconditionFailed); + return next(errors.PreconditionFailed, destBucketMD); + } + const { + storeMetadataParams, + error: metadataError, + sourceLocationConstraintName, + backendInfoDest, + } = _prepMetadata( + request, + sourceObjMD, + request.headers, + sourceIsDestination, + authInfo, + destObjectKey, + sourceBucketMD, + destBucketMD, + sourceVersionId, + log, + ); + if (metadataError) { + // eslint-disable-next-line no-param-reassign + request.sourceServerAccessLog && (request.sourceServerAccessLog.error = metadataError); + return next(metadataError, destBucketMD); + } + if (storeMetadataParams.metaHeaders) { + dataStoreContext.metaHeaders = storeMetadataParams.metaHeaders; + } - storeMetadataParams.overheadField = constants.overheadField; - - let dataLocator; - // If 0 byte object just set dataLocator to empty array - if (!sourceObjMD.location) { - dataLocator = []; - } else { - // To provide for backwards compatibility before - // md-model-version 2, need to handle cases where - // objMD.location is just a string - dataLocator = Array.isArray(sourceObjMD.location) ? - sourceObjMD.location : [{ key: sourceObjMD.location }]; - } + storeMetadataParams.overheadField = constants.overheadField; - if (sourceObjMD['x-amz-server-side-encryption']) { - for (let i = 0; i < dataLocator.length; i++) { - dataLocator[i].masterKeyId = sourceObjMD[ - 'x-amz-server-side-encryption-aws-kms-key-id']; - dataLocator[i].algorithm = - sourceObjMD['x-amz-server-side-encryption']; + let dataLocator; + // If 0 byte object just set dataLocator to empty array + if (!sourceObjMD.location) { + dataLocator = []; + } else { + // To provide for backwards compatibility before + // md-model-version 2, need to handle cases where + // objMD.location is just a string + dataLocator = Array.isArray(sourceObjMD.location) + ? sourceObjMD.location + : [{ key: sourceObjMD.location }]; + } + + if (sourceObjMD['x-amz-server-side-encryption']) { + for (let i = 0; i < dataLocator.length; i++) { + dataLocator[i].masterKeyId = sourceObjMD['x-amz-server-side-encryption-aws-kms-key-id']; + dataLocator[i].algorithm = sourceObjMD['x-amz-server-side-encryption']; + } } - } - // If the destination key already exists - if (destObjMD) { - // Re-use creation-time if we can - if (destObjMD['creation-time']) { - storeMetadataParams.creationTime = - destObjMD['creation-time']; - // Otherwise fallback to last-modified + // If the destination key already exists + if (destObjMD) { + // Re-use creation-time if we can + if (destObjMD['creation-time']) { + storeMetadataParams.creationTime = destObjMD['creation-time']; + // Otherwise fallback to last-modified + } else { + storeMetadataParams.creationTime = destObjMD['last-modified']; + } + // If this is a new key, create a new timestamp } else { - storeMetadataParams.creationTime = - destObjMD['last-modified']; + storeMetadataParams.creationTime = new Date().toJSON(); } - // If this is a new key, create a new timestamp - } else { - storeMetadataParams.creationTime = new Date().toJSON(); - } - return next(null, storeMetadataParams, dataLocator, - sourceBucketMD, destBucketMD, destObjMD, - sourceLocationConstraintName, backendInfoDest); - }); - }, - function getSSEConfiguration(storeMetadataParams, dataLocator, sourceBucketMD, - destBucketMD, destObjMD, sourceLocationConstraintName, - backendInfoDest, next) { - getObjectSSEConfiguration( - request.headers, - destBucketMD, - log, - (err, sseConfig) => - next(err, storeMetadataParams, dataLocator, sourceBucketMD, - destBucketMD, destObjMD, sourceLocationConstraintName, - backendInfoDest, sseConfig)); - }, - function goGetData(storeMetadataParams, dataLocator, sourceBucketMD, - destBucketMD, destObjMD, sourceLocationConstraintName, - backendInfoDest, serverSideEncryption, next) { - const vcfg = destBucketMD.getVersioningConfiguration(); - const isVersionedObj = vcfg && vcfg.Status === 'Enabled'; - const destLocationConstraintName = - storeMetadataParams.dataStoreName; - const needsEncryption = serverSideEncryption && !!serverSideEncryption.algo; - // skip if source and dest and location constraint the same and - // versioning is not enabled - // still send along serverSideEncryption info so algo - // and masterKeyId stored properly in metadata - if (sourceIsDestination && storeMetadataParams.locationMatch - && !isVersionedObj && !needsEncryption) { - return next(null, storeMetadataParams, dataLocator, destObjMD, - serverSideEncryption, destBucketMD); - } + return next( + null, + storeMetadataParams, + dataLocator, + sourceBucketMD, + destBucketMD, + destObjMD, + sourceLocationConstraintName, + backendInfoDest, + ); + }, + ); + }, + function getSSEConfiguration( + storeMetadataParams, + dataLocator, + sourceBucketMD, + destBucketMD, + destObjMD, + sourceLocationConstraintName, + backendInfoDest, + next, + ) { + getObjectSSEConfiguration(request.headers, destBucketMD, log, (err, sseConfig) => + next( + err, + storeMetadataParams, + dataLocator, + sourceBucketMD, + destBucketMD, + destObjMD, + sourceLocationConstraintName, + backendInfoDest, + sseConfig, + ), + ); + }, + function goGetData( + storeMetadataParams, + dataLocator, + sourceBucketMD, + destBucketMD, + destObjMD, + sourceLocationConstraintName, + backendInfoDest, + serverSideEncryption, + next, + ) { + const vcfg = destBucketMD.getVersioningConfiguration(); + const isVersionedObj = vcfg && vcfg.Status === 'Enabled'; + const destLocationConstraintName = storeMetadataParams.dataStoreName; + const needsEncryption = serverSideEncryption && !!serverSideEncryption.algo; + // skip if source and dest and location constraint the same and + // versioning is not enabled + // still send along serverSideEncryption info so algo + // and masterKeyId stored properly in metadata + if (sourceIsDestination && storeMetadataParams.locationMatch && !isVersionedObj && !needsEncryption) { + return next(null, storeMetadataParams, dataLocator, destObjMD, serverSideEncryption, destBucketMD); + } - // also skip if 0 byte object, unless location constraint is an - // external backend and differs from source, in which case put - // metadata to backend - let destLocationConstraintType; - if (config.backends.data === 'multiple') { - destLocationConstraintType = - config.getLocationConstraintType(destLocationConstraintName); - } - if (destLocationConstraintType && - versioningNotImplBackends[destLocationConstraintType] - && isVersionedObj) { - log.debug(externalVersioningErrorMessage, - { method: 'multipleBackendGateway', - error: errors.NotImplemented }); - return next(errorInstances.NotImplemented.customizeDescription( - externalVersioningErrorMessage), destBucketMD); - } - if (dataLocator.length === 0) { - if (!storeMetadataParams.locationMatch && - destLocationConstraintType && - constants.externalBackends[destLocationConstraintType]) { - return data.put(null, null, storeMetadataParams.size, - dataStoreContext, backendInfoDest, - log, (error, objectRetrievalInfo) => { - if (error) { - return next(error, destBucketMD); - } - const putResult = { - key: objectRetrievalInfo.key, - dataStoreName: objectRetrievalInfo. - dataStoreName, - dataStoreType: objectRetrievalInfo. - dataStoreType, - size: storeMetadataParams.size, - }; - const putResultArr = [putResult]; - return next(null, storeMetadataParams, putResultArr, - destObjMD, serverSideEncryption, destBucketMD); - }); + // also skip if 0 byte object, unless location constraint is an + // external backend and differs from source, in which case put + // metadata to backend + let destLocationConstraintType; + if (config.backends.data === 'multiple') { + destLocationConstraintType = config.getLocationConstraintType(destLocationConstraintName); } - return next(null, storeMetadataParams, dataLocator, destObjMD, - serverSideEncryption, destBucketMD); - } - const originalIdentityImpDenies = request.actionImplicitDenies; - // eslint-disable-next-line no-param-reassign - delete request.actionImplicitDenies; - return data.copyObject(request, sourceLocationConstraintName, - storeMetadataParams, dataLocator, dataStoreContext, - backendInfoDest, sourceBucketMD, destBucketMD, serverSideEncryption, log, - (err, results) => { - // eslint-disable-next-line no-param-reassign - request.actionImplicitDenies = originalIdentityImpDenies; - if (err) { - // eslint-disable-next-line no-param-reassign - request.sourceServerAccessLog && (request.sourceServerAccessLog.error = err); - return next(err, destBucketMD); + if ( + destLocationConstraintType && + versioningNotImplBackends[destLocationConstraintType] && + isVersionedObj + ) { + log.debug(externalVersioningErrorMessage, { + method: 'multipleBackendGateway', + error: errors.NotImplemented, + }); + return next( + errorInstances.NotImplemented.customizeDescription(externalVersioningErrorMessage), + destBucketMD, + ); } - return next(null, storeMetadataParams, results, - destObjMD, serverSideEncryption, destBucketMD); - }); - }, - function getVersioningInfo(storeMetadataParams, destDataGetInfoArr, - destObjMD, serverSideEncryption, destBucketMD, next) { - if (!destBucketMD.isVersioningEnabled() && destObjMD?.archive?.archiveInfo) { - // Ensure we trigger a "delete" event in the oplog for the previously archived object - // eslint-disable-next-line - storeMetadataParams.needOplogUpdate = 's3:ReplaceArchivedObject'; - } - return versioningPreprocessing(destBucketName, - destBucketMD, destObjectKey, destObjMD, log, - (err, options) => { - if (err) { - log.debug('error processing versioning info', - { error: err }); - return next(err, null, destBucketMD); + if (dataLocator.length === 0) { + if ( + !storeMetadataParams.locationMatch && + destLocationConstraintType && + constants.externalBackends[destLocationConstraintType] + ) { + return data.put( + null, + null, + storeMetadataParams.size, + dataStoreContext, + backendInfoDest, + log, + (error, objectRetrievalInfo) => { + if (error) { + return next(error, destBucketMD); + } + const putResult = { + key: objectRetrievalInfo.key, + dataStoreName: objectRetrievalInfo.dataStoreName, + dataStoreType: objectRetrievalInfo.dataStoreType, + size: storeMetadataParams.size, + }; + const putResultArr = [putResult]; + return next( + null, + storeMetadataParams, + putResultArr, + destObjMD, + serverSideEncryption, + destBucketMD, + ); + }, + ); } + return next(null, storeMetadataParams, dataLocator, destObjMD, serverSideEncryption, destBucketMD); + } + const originalIdentityImpDenies = request.actionImplicitDenies; + // eslint-disable-next-line no-param-reassign + delete request.actionImplicitDenies; + return data.copyObject( + request, + sourceLocationConstraintName, + storeMetadataParams, + dataLocator, + dataStoreContext, + backendInfoDest, + sourceBucketMD, + destBucketMD, + serverSideEncryption, + log, + (err, results) => { + // eslint-disable-next-line no-param-reassign + request.actionImplicitDenies = originalIdentityImpDenies; + if (err) { + // eslint-disable-next-line no-param-reassign + request.sourceServerAccessLog && (request.sourceServerAccessLog.error = err); + return next(err, destBucketMD); + } + return next(null, storeMetadataParams, results, destObjMD, serverSideEncryption, destBucketMD); + }, + ); + }, + function getVersioningInfo( + storeMetadataParams, + destDataGetInfoArr, + destObjMD, + serverSideEncryption, + destBucketMD, + next, + ) { + if (!destBucketMD.isVersioningEnabled() && destObjMD?.archive?.archiveInfo) { + // Ensure we trigger a "delete" event in the oplog for the previously archived object + // eslint-disable-next-line + storeMetadataParams.needOplogUpdate = 's3:ReplaceArchivedObject'; + } + return versioningPreprocessing( + destBucketName, + destBucketMD, + destObjectKey, + destObjMD, + log, + (err, options) => { + if (err) { + log.debug('error processing versioning info', { error: err }); + return next(err, null, destBucketMD); + } - const location = destDataGetInfoArr?.[0]?.dataStoreName; - if (location === destBucketMD.getLocationConstraint() && destBucketMD.isIngestionBucket()) { - // If the object is being written to the "ingested" storage location, keep the same - // versionId for consistency and to avoid creating an extra version when it gets - // ingested - const backendVersionId = decodeVID(destDataGetInfoArr[0].dataStoreVersionId); - if (!(backendVersionId instanceof Error)) { - options.versionId = backendVersionId; // eslint-disable-line no-param-reassign + const location = destDataGetInfoArr?.[0]?.dataStoreName; + if (location === destBucketMD.getLocationConstraint() && destBucketMD.isIngestionBucket()) { + // If the object is being written to the "ingested" storage location, keep the same + // versionId for consistency and to avoid creating an extra version when it gets + // ingested + const backendVersionId = decodeVID(destDataGetInfoArr[0].dataStoreVersionId); + if (!(backendVersionId instanceof Error)) { + options.versionId = backendVersionId; // eslint-disable-line no-param-reassign + } } - } + // eslint-disable-next-line + storeMetadataParams.versionId = options.versionId; + // eslint-disable-next-line + storeMetadataParams.versioning = options.versioning; + // eslint-disable-next-line + storeMetadataParams.isNull = options.isNull; + if (options.extraMD) { + Object.assign(storeMetadataParams, options.extraMD); + } + const dataToDelete = options.dataToDelete; + return next( + null, + storeMetadataParams, + destDataGetInfoArr, + destObjMD, + serverSideEncryption, + destBucketMD, + dataToDelete, + ); + }, + ); + }, + function storeNewMetadata( + storeMetadataParams, + destDataGetInfoArr, + destObjMD, + serverSideEncryption, + destBucketMD, + dataToDelete, + next, + ) { + if (destObjMD && destObjMD.uploadId) { // eslint-disable-next-line - storeMetadataParams.versionId = options.versionId; - // eslint-disable-next-line - storeMetadataParams.versioning = options.versioning; - // eslint-disable-next-line - storeMetadataParams.isNull = options.isNull; - if (options.extraMD) { - Object.assign(storeMetadataParams, options.extraMD); - } - const dataToDelete = options.dataToDelete; - return next(null, storeMetadataParams, destDataGetInfoArr, - destObjMD, serverSideEncryption, destBucketMD, - dataToDelete); - }); - }, - function storeNewMetadata(storeMetadataParams, destDataGetInfoArr, - destObjMD, serverSideEncryption, destBucketMD, dataToDelete, next) { - if (destObjMD && destObjMD.uploadId) { - // eslint-disable-next-line - storeMetadataParams.oldReplayId = destObjMD.uploadId; - } + storeMetadataParams.oldReplayId = destObjMD.uploadId; + } - return services.metadataStoreObject(destBucketName, - destDataGetInfoArr, serverSideEncryption, - storeMetadataParams, (err, result) => { - if (err) { - log.debug('error storing new metadata', { error: err }); - return next(err, null, destBucketMD); - } - const sourceObjSize = storeMetadataParams.size; - const destObjPrevSize = (destObjMD && - destObjMD['content-length'] !== undefined) ? - destObjMD['content-length'] : null; - - setExpirationHeaders(responseHeaders, { - lifecycleConfig: destBucketMD.getLifecycleConfiguration(), - objectParams: { - key: destObjectKey, - date: result.lastModified, - tags: result.tags, - }, - }); + return services.metadataStoreObject( + destBucketName, + destDataGetInfoArr, + serverSideEncryption, + storeMetadataParams, + (err, result) => { + if (err) { + log.debug('error storing new metadata', { error: err }); + return next(err, null, destBucketMD); + } + const sourceObjSize = storeMetadataParams.size; + const destObjPrevSize = + destObjMD && destObjMD['content-length'] !== undefined ? destObjMD['content-length'] : null; + + setExpirationHeaders(responseHeaders, { + lifecycleConfig: destBucketMD.getLifecycleConfiguration(), + objectParams: { + key: destObjectKey, + date: result.lastModified, + tags: result.tags, + }, + }); - return next(null, dataToDelete, result, destBucketMD, - storeMetadataParams, serverSideEncryption, - sourceObjSize, destObjPrevSize); - }); - }, - function deleteExistingData(dataToDelete, storingNewMdResult, - destBucketMD, storeMetadataParams, serverSideEncryption, - sourceObjSize, destObjPrevSize, next) { - // Clean up any potential orphans in data if object - // put is an overwrite of already existing - // object with same name, so long as the source is not - // the same as the destination - if (!sourceIsDestination && dataToDelete) { - const newDataStoreName = storeMetadataParams.dataStoreName; - return data.batchDelete(dataToDelete, request.method, - newDataStoreName, log, err => { + return next( + null, + dataToDelete, + result, + destBucketMD, + storeMetadataParams, + serverSideEncryption, + sourceObjSize, + destObjPrevSize, + ); + }, + ); + }, + function deleteExistingData( + dataToDelete, + storingNewMdResult, + destBucketMD, + storeMetadataParams, + serverSideEncryption, + sourceObjSize, + destObjPrevSize, + next, + ) { + // Clean up any potential orphans in data if object + // put is an overwrite of already existing + // object with same name, so long as the source is not + // the same as the destination + if (!sourceIsDestination && dataToDelete) { + const newDataStoreName = storeMetadataParams.dataStoreName; + return data.batchDelete(dataToDelete, request.method, newDataStoreName, log, err => { if (err) { // if error, log the error and move on as it is not // relevant to the client as the client's // object already succeeded putting data, metadata - log.error('error deleting existing data', - { error: err }); + log.error('error deleting existing data', { error: err }); } - next(null, - storingNewMdResult, destBucketMD, storeMetadataParams, - serverSideEncryption, sourceObjSize, destObjPrevSize); + next( + null, + storingNewMdResult, + destBucketMD, + storeMetadataParams, + serverSideEncryption, + sourceObjSize, + destObjPrevSize, + ); }); + } + return next( + null, + storingNewMdResult, + destBucketMD, + storeMetadataParams, + serverSideEncryption, + sourceObjSize, + destObjPrevSize, + ); + }, + ], + ( + err, + storingNewMdResult, + destBucketMD, + storeMetadataParams, + serverSideEncryption, + sourceObjSize, + destObjPrevSize, + ) => { + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, destBucketMD); + + // Store full object size for server access logs + if (request.serverAccessLog) { + // eslint-disable-next-line no-param-reassign + request.serverAccessLog.objectSize = sourceObjSize; + } + + // Initialize the queue for internal log request logging + initializeInternalLogRequestQueue(request); + // Queue the source-side access log (REST.COPY.OBJECT_GET) + queueInternalLogRequest(request, { + operation: 'REST.COPY.OBJECT_GET', + sourceBucket, + sourceObject, + objectSize: sourceObjSize || null, + }); + + if (err) { + monitoring.promMetrics('PUT', destBucketName, err.code, 'copyObject'); + return callback(err, null, corsHeaders); + } + const xml = [ + '', + '', + '', + new Date(storeMetadataParams.lastModifiedDate).toISOString(), + '', + '"', + storeMetadataParams.contentMD5, + '"', + '', + ].join(''); + const additionalHeaders = corsHeaders || {}; + if (serverSideEncryption) { + setSSEHeaders( + additionalHeaders, + serverSideEncryption.algorithm, + serverSideEncryption.configuredMasterKeyId || serverSideEncryption.masterKeyId, + ); + } + if (sourceVersionId) { + additionalHeaders['x-amz-copy-source-version-id'] = versionIdUtils.encode(sourceVersionId); + } + const isVersioned = storingNewMdResult && storingNewMdResult.versionId; + if (isVersioned) { + additionalHeaders['x-amz-version-id'] = versionIdUtils.encode(storingNewMdResult.versionId); } - return next(null, - storingNewMdResult, destBucketMD, storeMetadataParams, - serverSideEncryption, sourceObjSize, destObjPrevSize); - }, - ], (err, storingNewMdResult, destBucketMD, storeMetadataParams, - serverSideEncryption, sourceObjSize, destObjPrevSize) => { - const corsHeaders = collectCorsHeaders(request.headers.origin, - request.method, destBucketMD); - - // Store full object size for server access logs - if (request.serverAccessLog) { - // eslint-disable-next-line no-param-reassign - request.serverAccessLog.objectSize = sourceObjSize; - } - - // Initialize the queue for internal log request logging - initializeInternalLogRequestQueue(request); - // Queue the source-side access log (REST.COPY.OBJECT_GET) - queueInternalLogRequest(request, { - operation: 'REST.COPY.OBJECT_GET', - sourceBucket, - sourceObject, - objectSize: sourceObjSize || null, - }); - if (err) { + Object.assign(responseHeaders, additionalHeaders); + + // Only pre-existing non-versioned objects get 0 all others use 1 + const numberOfObjects = !isVersioned && destObjPrevSize !== null ? 0 : 1; + + pushMetric('copyObject', log, { + authInfo, + canonicalID: destBucketMD.getOwner(), + bucket: destBucketName, + keys: [destObjectKey], + newByteLength: sourceObjSize, + oldByteLength: isVersioned ? null : destObjPrevSize, + location: storeMetadataParams.dataStoreName, + versionId: isVersioned ? storingNewMdResult.versionId : undefined, + numberOfObjects, + }); monitoring.promMetrics( - 'PUT', destBucketName, err.code, 'copyObject'); - return callback(err, null, corsHeaders); - } - const xml = [ - '', - '', - '', new Date(storeMetadataParams.lastModifiedDate) - .toISOString(), '', - '"', storeMetadataParams.contentMD5, '"', - '', - ].join(''); - const additionalHeaders = corsHeaders || {}; - if (serverSideEncryption) { - setSSEHeaders(additionalHeaders, - serverSideEncryption.algorithm, - serverSideEncryption.configuredMasterKeyId || serverSideEncryption.masterKeyId + 'PUT', + destBucketName, + '200', + 'copyObject', + sourceObjSize, + destObjPrevSize, + isVersioned, ); - } - if (sourceVersionId) { - additionalHeaders['x-amz-copy-source-version-id'] = - versionIdUtils.encode(sourceVersionId); - } - const isVersioned = storingNewMdResult && storingNewMdResult.versionId; - if (isVersioned) { - additionalHeaders['x-amz-version-id'] = - versionIdUtils.encode(storingNewMdResult.versionId); - } - - Object.assign(responseHeaders, additionalHeaders); - - // Only pre-existing non-versioned objects get 0 all others use 1 - const numberOfObjects = !isVersioned && destObjPrevSize !== null ? 0 : 1; - - pushMetric('copyObject', log, { - authInfo, - canonicalID: destBucketMD.getOwner(), - bucket: destBucketName, - keys: [destObjectKey], - newByteLength: sourceObjSize, - oldByteLength: isVersioned ? null : destObjPrevSize, - location: storeMetadataParams.dataStoreName, - versionId: isVersioned ? storingNewMdResult.versionId : undefined, - numberOfObjects, - }); - monitoring.promMetrics('PUT', destBucketName, '200', - 'copyObject', sourceObjSize, destObjPrevSize, isVersioned); - // Add expiration header if lifecycle enabled - return callback(null, xml, responseHeaders); - }); + // Add expiration header if lifecycle enabled + return callback(null, xml, responseHeaders); + }, + ); } module.exports = objectCopy; diff --git a/lib/api/objectDelete.js b/lib/api/objectDelete.js index a1901f4a5f..42375420b9 100644 --- a/lib/api/objectDelete.js +++ b/lib/api/objectDelete.js @@ -5,12 +5,10 @@ const collectCorsHeaders = require('../utilities/collectCorsHeaders'); const services = require('../services'); const { pushMetric } = require('../utapi/utilities'); const createAndStoreObject = require('./apiUtils/object/createAndStoreObject'); -const { decodeVersionId, preprocessingVersioningDelete } - = require('./apiUtils/object/versioning'); +const { decodeVersionId, preprocessingVersioningDelete } = require('./apiUtils/object/versioning'); const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUtils'); const monitoring = require('../utilities/monitoringHandler'); -const { hasGovernanceBypassHeader, ObjectLockInfo } - = require('./apiUtils/object/objectLockHelpers'); +const { hasGovernanceBypassHeader, ObjectLockInfo } = require('./apiUtils/object/objectLockHelpers'); const { config } = require('../Config'); const { _deleteRequiresOplogUpdate } = require('./apiUtils/object/deleteObject'); @@ -32,8 +30,7 @@ function objectDeleteInternal(authInfo, request, log, isExpiration, cb) { log.debug('processing request', { method: 'objectDeleteInternal' }); if (authInfo.isRequesterPublicUser()) { log.debug('operation not available for public user'); - monitoring.promMetrics( - 'DELETE', request.bucketName, 403, 'deleteObject'); + monitoring.promMetrics('DELETE', request.bucketName, 403, 'deleteObject'); return cb(errors.AccessDenied); } const bucketName = request.bucketName; @@ -60,253 +57,303 @@ function objectDeleteInternal(authInfo, request, log, isExpiration, cb) { }; const canonicalID = authInfo.getCanonicalID(); - return async.waterfall([ - function validateBucketAndObj(next) { - return standardMetadataValidateBucketAndObj(valParams, request.actionImplicitDenies, log, - (err, bucketMD, objMD) => { - if (err) { - return next(err, bucketMD); + return async.waterfall( + [ + function validateBucketAndObj(next) { + return standardMetadataValidateBucketAndObj( + valParams, + request.actionImplicitDenies, + log, + (err, bucketMD, objMD) => { + if (err) { + return next(err, bucketMD); + } + + const versioningCfg = bucketMD.getVersioningConfiguration(); + if (!objMD) { + if (!versioningCfg) { + return next(errors.NoSuchKey, bucketMD); + } + // AWS does not return an error when trying to delete a + // specific version that does not exist. We skip to the end + // of the waterfall here. + if (reqVersionId) { + log.debug('trying to delete specific version ' + ' that does not exist'); + return next(errors.NoSuchVersion, bucketMD); + } + // To adhere to AWS behavior, create a delete marker even + // if trying to delete an object that does not exist when + // versioning has been configured + return next(null, bucketMD, objMD); + } + + if ( + versioningCfg && + versioningCfg.Status === 'Enabled' && + objMD.versionId === reqVersionId && + isExpiration && + !objMD.isDeleteMarker + ) { + log.warn( + 'expiration is trying to delete a master version ' + + 'of an object with versioning enabled', + { + method: 'objectDeleteInternal', + isExpiration, + reqVersionId, + versionId: objMD.versionId, + replicationState: objMD.replicationInfo, + location: objMD.location, + originOp: objMD.originOp, + }, + ); + } + if (reqVersionId && objMD.location && Array.isArray(objMD.location) && objMD.location[0]) { + // we need this information for data deletes to AWS + // eslint-disable-next-line no-param-reassign + objMD.location[0].deleteVersion = true; + } + if (objMD['content-length'] !== undefined) { + log.end().addDefaultFields({ + bytesDeleted: objMD['content-length'], + }); + // Store full object size for server access logs + if (request.serverAccessLog) { + // eslint-disable-next-line no-param-reassign + request.serverAccessLog.analyticsBytesDeleted = objMD['content-length']; + // eslint-disable-next-line no-param-reassign + request.serverAccessLog.objectSize = parseInt(objMD['content-length'], 10); + } + } + return next(null, bucketMD, objMD); + }, + ); + }, + function evaluateObjectLockPolicy(bucketMD, objectMD, next) { + // AWS only returns an object lock error if a version id + // is specified, else continue to create a delete marker + if (!reqVersionId) { + return next(null, bucketMD, objectMD); } - const versioningCfg = bucketMD.getVersioningConfiguration(); - if (!objMD) { - if (!versioningCfg) { - return next(errors.NoSuchKey, bucketMD); - } - // AWS does not return an error when trying to delete a - // specific version that does not exist. We skip to the end - // of the waterfall here. - if (reqVersionId) { - log.debug('trying to delete specific version ' + - ' that does not exist'); - return next(errors.NoSuchVersion, bucketMD); - } - // To adhere to AWS behavior, create a delete marker even - // if trying to delete an object that does not exist when - // versioning has been configured - return next(null, bucketMD, objMD); + const objLockInfo = new ObjectLockInfo({ + mode: objectMD.retentionMode, + date: objectMD.retentionDate, + legalHold: objectMD.legalHold || false, + }); + + // If the object can not be deleted raise an error + if (!objLockInfo.canModifyObject(hasGovernanceBypass)) { + log.debug('trying to delete locked object'); + return next(objectLockedError, bucketMD); } - if (versioningCfg && versioningCfg.Status === 'Enabled' && - objMD.versionId === reqVersionId && isExpiration && - !objMD.isDeleteMarker) { - log.warn('expiration is trying to delete a master version ' + - 'of an object with versioning enabled', { - method: 'objectDeleteInternal', - isExpiration, - reqVersionId, - versionId: objMD.versionId, - replicationState: objMD.replicationInfo, - location: objMD.location, - originOp: objMD.originOp, - }); + return next(null, bucketMD, objectMD); + }, + function validateHeaders(bucketMD, objectMD, next) { + if (objectMD) { + const lastModified = objectMD['last-modified']; + const { modifiedSinceRes, unmodifiedSinceRes } = checkDateModifiedHeaders( + request.headers, + lastModified, + ); + const err = modifiedSinceRes.error || unmodifiedSinceRes.error; + if (err) { + return process.nextTick(() => next(err, bucketMD)); } - if (reqVersionId && objMD.location && - Array.isArray(objMD.location) && objMD.location[0]) { - // we need this information for data deletes to AWS - // eslint-disable-next-line no-param-reassign - objMD.location[0].deleteVersion = true; } - if (objMD['content-length'] !== undefined) { - log.end().addDefaultFields({ - bytesDeleted: objMD['content-length'], - }); - // Store full object size for server access logs - if (request.serverAccessLog) { - // eslint-disable-next-line no-param-reassign - request.serverAccessLog.analyticsBytesDeleted = objMD['content-length']; - // eslint-disable-next-line no-param-reassign - request.serverAccessLog.objectSize = parseInt(objMD['content-length'], 10); - } + return process.nextTick(() => next(null, bucketMD, objectMD)); + }, + function deleteOperation(bucketMD, objectMD, next) { + const delOptions = preprocessingVersioningDelete( + bucketName, + bucketMD, + objectMD, + reqVersionId, + config.nullVersionCompatMode, + ); + const deleteInfo = { + removeDeleteMarker: false, + newDeleteMarker: false, + }; + if (delOptions && delOptions.deleteData && bucketMD.isNFS() && bucketMD.getReplicationConfiguration()) { + // If an NFS bucket that has replication configured, we want + // to put a delete marker on the destination even though the + // source does not have versioning. + return createAndStoreObject( + bucketName, + bucketMD, + objectKey, + objectMD, + authInfo, + canonicalID, + null, + request, + true, + null, + log, + isExpiration + ? 's3:LifecycleExpiration:DeleteMarkerCreated' + : 's3:ObjectRemoved:DeleteMarkerCreated', + err => { + if (err) { + return next(err); + } + if (objectMD.isDeleteMarker) { + // record that we deleted a delete marker to set + // response headers accordingly + deleteInfo.removeDeleteMarker = true; + } + return services.deleteObject( + bucketName, + objectMD, + objectKey, + delOptions, + false, + log, + isExpiration ? 's3:LifecycleExpiration:Delete' : 's3:ObjectRemoved:Delete', + (err, delResult) => next(err, bucketMD, objectMD, delResult, deleteInfo), + ); + }, + ); } - return next(null, bucketMD, objMD); - }); - }, - function evaluateObjectLockPolicy(bucketMD, objectMD, next) { - // AWS only returns an object lock error if a version id - // is specified, else continue to create a delete marker - if (!reqVersionId) { - return next(null, bucketMD, objectMD); - } - - const objLockInfo = new ObjectLockInfo({ - mode: objectMD.retentionMode, - date: objectMD.retentionDate, - legalHold: objectMD.legalHold || false, - }); + if (delOptions && delOptions.deleteData) { + delOptions.overheadField = overheadField; + if (objectMD.isDeleteMarker) { + // record that we deleted a delete marker to set + // response headers accordingly + deleteInfo.removeDeleteMarker = true; + } - // If the object can not be deleted raise an error - if (!objLockInfo.canModifyObject(hasGovernanceBypass)) { - log.debug('trying to delete locked object'); - return next(objectLockedError, bucketMD); - } + if (objectMD.uploadId) { + delOptions.replayId = objectMD.uploadId; + } - return next(null, bucketMD, objectMD); - }, - function validateHeaders(bucketMD, objectMD, next) { - if (objectMD) { - const lastModified = objectMD['last-modified']; - const { modifiedSinceRes, unmodifiedSinceRes } = - checkDateModifiedHeaders(request.headers, lastModified); - const err = modifiedSinceRes.error || unmodifiedSinceRes.error; - if (err) { - return process.nextTick(() => next(err, bucketMD)); - } - } - return process.nextTick(() => - next(null, bucketMD, objectMD)); - }, - function deleteOperation(bucketMD, objectMD, next) { - const delOptions = preprocessingVersioningDelete( - bucketName, bucketMD, objectMD, reqVersionId, config.nullVersionCompatMode); - const deleteInfo = { - removeDeleteMarker: false, - newDeleteMarker: false, - }; - if (delOptions && delOptions.deleteData && bucketMD.isNFS() && - bucketMD.getReplicationConfiguration()) { - // If an NFS bucket that has replication configured, we want - // to put a delete marker on the destination even though the - // source does not have versioning. - return createAndStoreObject(bucketName, bucketMD, objectKey, - objectMD, authInfo, canonicalID, null, request, true, null, - log, isExpiration ? - 's3:LifecycleExpiration:DeleteMarkerCreated' : - 's3:ObjectRemoved:DeleteMarkerCreated', - err => { - if (err) { - return next(err); - } - if (objectMD.isDeleteMarker) { - // record that we deleted a delete marker to set - // response headers accordingly - deleteInfo.removeDeleteMarker = true; - } - return services.deleteObject(bucketName, objectMD, - objectKey, delOptions, false, log, isExpiration ? - 's3:LifecycleExpiration:Delete' : - 's3:ObjectRemoved:Delete', - (err, delResult) => - next(err, bucketMD, objectMD, delResult, deleteInfo)); - }); - } - if (delOptions && delOptions.deleteData) { - delOptions.overheadField = overheadField; - if (objectMD.isDeleteMarker) { - // record that we deleted a delete marker to set - // response headers accordingly - deleteInfo.removeDeleteMarker = true; - } + if (!_deleteRequiresOplogUpdate(objectMD, bucketMD)) { + delOptions.doesNotNeedOpogUpdate = true; + } - if (objectMD.uploadId) { - delOptions.replayId = objectMD.uploadId; + return services.deleteObject( + bucketName, + objectMD, + objectKey, + delOptions, + false, + log, + isExpiration ? 's3:LifecycleExpiration:Delete' : 's3:ObjectRemoved:Delete', + (err, delResult) => next(err, bucketMD, objectMD, delResult, deleteInfo), + ); } - - if (!_deleteRequiresOplogUpdate(objectMD, bucketMD)) { - delOptions.doesNotNeedOpogUpdate = true; + // putting a new delete marker + deleteInfo.newDeleteMarker = true; + return createAndStoreObject( + bucketName, + bucketMD, + objectKey, + objectMD, + authInfo, + canonicalID, + null, + request, + deleteInfo.newDeleteMarker, + null, + overheadField, + log, + isExpiration + ? 's3:LifecycleExpiration:DeleteMarkerCreated' + : 's3:ObjectRemoved:DeleteMarkerCreated', + (err, newDelMarkerRes) => { + next(err, bucketMD, objectMD, newDelMarkerRes, deleteInfo); + }, + ); + }, + ], + (err, bucketMD, objectMD, result, deleteInfo) => { + const resHeaders = collectCorsHeaders(request.headers.origin, request.method, bucketMD); + // if deleting a specific version or delete marker, return version id + // in the response headers, even in case of NoSuchVersion + if (reqVersionId) { + resHeaders['x-amz-version-id'] = + reqVersionId === 'null' ? reqVersionId : versionIdUtils.encode(reqVersionId); + if (deleteInfo && deleteInfo.removeDeleteMarker) { + resHeaders['x-amz-delete-marker'] = true; } - - return services.deleteObject(bucketName, objectMD, objectKey, - delOptions, false, log, isExpiration ? - 's3:LifecycleExpiration:Delete' : - 's3:ObjectRemoved:Delete', - (err, delResult) => next(err, bucketMD, - objectMD, delResult, deleteInfo)); - } - // putting a new delete marker - deleteInfo.newDeleteMarker = true; - return createAndStoreObject(bucketName, bucketMD, - objectKey, objectMD, authInfo, canonicalID, null, request, - deleteInfo.newDeleteMarker, null, overheadField, log, isExpiration ? - 's3:LifecycleExpiration:DeleteMarkerCreated' : - 's3:ObjectRemoved:DeleteMarkerCreated', - (err, newDelMarkerRes) => { - next(err, bucketMD, objectMD, newDelMarkerRes, deleteInfo); - }); - }, - ], (err, bucketMD, objectMD, result, deleteInfo) => { - const resHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucketMD); - // if deleting a specific version or delete marker, return version id - // in the response headers, even in case of NoSuchVersion - if (reqVersionId) { - resHeaders['x-amz-version-id'] = reqVersionId === 'null' ? - reqVersionId : versionIdUtils.encode(reqVersionId); - if (deleteInfo && deleteInfo.removeDeleteMarker) { - resHeaders['x-amz-delete-marker'] = true; } - } - if (err === objectLockedError) { - log.debug('preventing deletion due to object lock', - { + if (err === objectLockedError) { + log.debug('preventing deletion due to object lock', { error: errors.AccessDenied, objectLocked: true, method: 'objectDelete', }); - return cb(errors.AccessDenied, resHeaders); - } - if (err) { - log.debug('error processing request', { error: err, - method: 'objectDelete' }); - monitoring.promMetrics( - 'DELETE', bucketName, err.code, 'deleteObject'); - return cb(err, resHeaders); - } - if (deleteInfo.newDeleteMarker) { - // if we created a new delete marker, return true for - // x-amz-delete-marker and the version ID of the new delete marker - if (result.versionId) { - resHeaders['x-amz-delete-marker'] = true; - resHeaders['x-amz-version-id'] = result.versionId === 'null' ? - result.versionId : versionIdUtils.encode(result.versionId); + return cb(errors.AccessDenied, resHeaders); } + if (err) { + log.debug('error processing request', { error: err, method: 'objectDelete' }); + monitoring.promMetrics('DELETE', bucketName, err.code, 'deleteObject'); + return cb(err, resHeaders); + } + if (deleteInfo.newDeleteMarker) { + // if we created a new delete marker, return true for + // x-amz-delete-marker and the version ID of the new delete marker + if (result.versionId) { + resHeaders['x-amz-delete-marker'] = true; + resHeaders['x-amz-version-id'] = + result.versionId === 'null' ? result.versionId : versionIdUtils.encode(result.versionId); + } - /* byteLength is passed under the following conditions: - * - bucket versioning is suspended - * - object version id is null - * and one of: - * - the content length of the object exists - * - or - - * - it is a delete marker - * In this case, the master key is deleted and replaced with a delete marker. - * The decrement accounts for the deletion of the master key when utapi reports - * on the number of objects. - */ - // FIXME: byteLength may be incorrect, see S3C-7440 - const versioningSuspended = bucketMD.getVersioningConfiguration() - && bucketMD.getVersioningConfiguration().Status === 'Suspended'; - const deletedSuspendedMasterVersion = versioningSuspended && !!objectMD; - // Default to 0 content-length to cover deleting a DeleteMarker - const objectByteLength = (objectMD && objectMD['content-length']) || 0; - const byteLength = deletedSuspendedMasterVersion ? Number.parseInt(objectByteLength, 10) : null; + /* byteLength is passed under the following conditions: + * - bucket versioning is suspended + * - object version id is null + * and one of: + * - the content length of the object exists + * - or - + * - it is a delete marker + * In this case, the master key is deleted and replaced with a delete marker. + * The decrement accounts for the deletion of the master key when utapi reports + * on the number of objects. + */ + // FIXME: byteLength may be incorrect, see S3C-7440 + const versioningSuspended = + bucketMD.getVersioningConfiguration() && + bucketMD.getVersioningConfiguration().Status === 'Suspended'; + const deletedSuspendedMasterVersion = versioningSuspended && !!objectMD; + // Default to 0 content-length to cover deleting a DeleteMarker + const objectByteLength = (objectMD && objectMD['content-length']) || 0; + const byteLength = deletedSuspendedMasterVersion ? Number.parseInt(objectByteLength, 10) : null; - pushMetric('putDeleteMarkerObject', log, { - authInfo, - byteLength, - bucket: bucketName, - keys: [objectKey], - versionId: result.versionId, - location: objectMD ? objectMD.dataStoreName : undefined, - }); - } else { - log.end().addDefaultFields({ - contentLength: objectMD['content-length'], - }); - pushMetric('deleteObject', log, { - authInfo, - canonicalID: bucketMD.getOwner(), - bucket: bucketName, - keys: [objectKey], - byteLength: Number.parseInt(objectMD['content-length'], 10), - numberOfObjects: 1, - location: objectMD.dataStoreName, - isDelete: true, - }); - monitoring.promMetrics('DELETE', bucketName, '200', 'deleteObject', - Number.parseInt(objectMD['content-length'], 10)); - } - return cb(err, resHeaders); - }); + pushMetric('putDeleteMarkerObject', log, { + authInfo, + byteLength, + bucket: bucketName, + keys: [objectKey], + versionId: result.versionId, + location: objectMD ? objectMD.dataStoreName : undefined, + }); + } else { + log.end().addDefaultFields({ + contentLength: objectMD['content-length'], + }); + pushMetric('deleteObject', log, { + authInfo, + canonicalID: bucketMD.getOwner(), + bucket: bucketName, + keys: [objectKey], + byteLength: Number.parseInt(objectMD['content-length'], 10), + numberOfObjects: 1, + location: objectMD.dataStoreName, + isDelete: true, + }); + monitoring.promMetrics( + 'DELETE', + bucketName, + '200', + 'deleteObject', + Number.parseInt(objectMD['content-length'], 10), + ); + } + return cb(err, resHeaders); + }, + ); } /** diff --git a/lib/api/objectDeleteTagging.js b/lib/api/objectDeleteTagging.js index 71115ffe5a..45d67a0f97 100644 --- a/lib/api/objectDeleteTagging.js +++ b/lib/api/objectDeleteTagging.js @@ -1,8 +1,11 @@ const async = require('async'); const { errors } = require('arsenal'); -const { decodeVersionId, getVersionIdResHeader, getVersionSpecificMetadataOptions } - = require('./apiUtils/object/versioning'); +const { + decodeVersionId, + getVersionIdResHeader, + getVersionSpecificMetadataOptions, +} = require('./apiUtils/object/versioning'); const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUtils'); const { pushMetric } = require('../utapi/utilities'); @@ -48,75 +51,81 @@ function objectDeleteTagging(authInfo, request, log, callback) { request, }; - return async.waterfall([ - next => standardMetadataValidateBucketAndObj(metadataValParams, request.actionImplicitDenies, log, - (err, bucket, objectMD) => { - if (err) { - log.trace('request authorization failed', - { method: 'objectDeleteTagging', error: err }); - return next(err); - } - if (!objectMD) { - const err = reqVersionId ? errors.NoSuchVersion : - errors.NoSuchKey; - log.trace('error no object metadata found', - { method: 'objectDeleteTagging', error: err }); - return next(err, bucket); - } - if (objectMD.isDeleteMarker) { - log.trace('version is a delete marker', - { method: 'objectDeleteTagging' }); - // FIXME we should return a `x-amz-delete-marker: true` header, - // see S3C-7592 - return next(errors.MethodNotAllowed, bucket); - } - return next(null, bucket, objectMD); - }), - (bucket, objectMD, next) => { - // eslint-disable-next-line no-param-reassign - objectMD.tags = {}; - const params = getVersionSpecificMetadataOptions(objectMD, config.nullVersionCompatMode); - const replicationInfo = getReplicationInfo(config, - objectKey, bucket, true, 0, REPLICATION_ACTION, objectMD); - if (replicationInfo) { + return async.waterfall( + [ + next => + standardMetadataValidateBucketAndObj( + metadataValParams, + request.actionImplicitDenies, + log, + (err, bucket, objectMD) => { + if (err) { + log.trace('request authorization failed', { method: 'objectDeleteTagging', error: err }); + return next(err); + } + if (!objectMD) { + const err = reqVersionId ? errors.NoSuchVersion : errors.NoSuchKey; + log.trace('error no object metadata found', { method: 'objectDeleteTagging', error: err }); + return next(err, bucket); + } + if (objectMD.isDeleteMarker) { + log.trace('version is a delete marker', { method: 'objectDeleteTagging' }); + // FIXME we should return a `x-amz-delete-marker: true` header, + // see S3C-7592 + return next(errors.MethodNotAllowed, bucket); + } + return next(null, bucket, objectMD); + }, + ), + (bucket, objectMD, next) => { // eslint-disable-next-line no-param-reassign - objectMD.replicationInfo = Object.assign({}, - objectMD.replicationInfo, replicationInfo); + objectMD.tags = {}; + const params = getVersionSpecificMetadataOptions(objectMD, config.nullVersionCompatMode); + const replicationInfo = getReplicationInfo( + config, + objectKey, + bucket, + true, + 0, + REPLICATION_ACTION, + objectMD, + ); + if (replicationInfo) { + // eslint-disable-next-line no-param-reassign + objectMD.replicationInfo = Object.assign({}, objectMD.replicationInfo, replicationInfo); + } + // eslint-disable-next-line no-param-reassign + objectMD.originOp = 's3:ObjectTagging:Delete'; + metadata.putObjectMD(bucket.getName(), objectKey, objectMD, params, log, err => + next(err, bucket, objectMD), + ); + }, + (bucket, objectMD, next) => + // if external backends handles tagging + data.objectTagging('Delete', objectKey, bucket.getName(), objectMD, log, err => + next(err, bucket, objectMD), + ), + ], + (err, bucket, objectMD) => { + const additionalResHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + if (err) { + log.trace('error processing request', { error: err, method: 'objectDeleteTagging' }); + monitoring.promMetrics('DELETE', bucketName, err.code, 'deleteObjectTagging'); + } else { + pushMetric('deleteObjectTagging', log, { + authInfo, + bucket: bucketName, + keys: [objectKey], + versionId: objectMD ? objectMD.versionId : undefined, + location: objectMD ? objectMD.dataStoreName : undefined, + }); + monitoring.promMetrics('DELETE', bucketName, '200', 'deleteObjectTagging'); + const verCfg = bucket.getVersioningConfiguration(); + additionalResHeaders['x-amz-version-id'] = getVersionIdResHeader(verCfg, objectMD); } - // eslint-disable-next-line no-param-reassign - objectMD.originOp = 's3:ObjectTagging:Delete'; - metadata.putObjectMD(bucket.getName(), objectKey, objectMD, params, - log, err => - next(err, bucket, objectMD)); + return callback(err, additionalResHeaders); }, - (bucket, objectMD, next) => - // if external backends handles tagging - data.objectTagging('Delete', objectKey, bucket.getName(), objectMD, - log, err => next(err, bucket, objectMD)), - ], (err, bucket, objectMD) => { - const additionalResHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); - if (err) { - log.trace('error processing request', { error: err, - method: 'objectDeleteTagging' }); - monitoring.promMetrics( - 'DELETE', bucketName, err.code, 'deleteObjectTagging'); - } else { - pushMetric('deleteObjectTagging', log, { - authInfo, - bucket: bucketName, - keys: [objectKey], - versionId: objectMD ? objectMD.versionId : undefined, - location: objectMD ? objectMD.dataStoreName : undefined, - }); - monitoring.promMetrics( - 'DELETE', bucketName, '200', 'deleteObjectTagging'); - const verCfg = bucket.getVersioningConfiguration(); - additionalResHeaders['x-amz-version-id'] = - getVersionIdResHeader(verCfg, objectMD); - } - return callback(err, additionalResHeaders); - }); + ); } module.exports = objectDeleteTagging; diff --git a/lib/api/objectGet.js b/lib/api/objectGet.js index 833dd0d7c0..0a1dab00cf 100644 --- a/lib/api/objectGet.js +++ b/lib/api/objectGet.js @@ -10,10 +10,8 @@ const collectResponseHeaders = require('../utilities/collectResponseHeaders'); const { pushMetric } = require('../utapi/utilities'); const { getVersionIdResHeader } = require('./apiUtils/object/versioning'); const setPartRanges = require('./apiUtils/object/setPartRanges'); -const locationHeaderCheck = - require('./apiUtils/object/locationHeaderCheck'); -const getReplicationBackendDataLocator = - require('./apiUtils/object/getReplicationBackendDataLocator'); +const locationHeaderCheck = require('./apiUtils/object/locationHeaderCheck'); +const getReplicationBackendDataLocator = require('./apiUtils/object/getReplicationBackendDataLocator'); const checkReadLocation = require('./apiUtils/object/checkReadLocation'); const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUtils'); @@ -42,8 +40,7 @@ function objectGet(authInfo, request, returnTagCount, log, callback) { const objectKey = request.objectKey; // returns name of location to get from and key if successful - const locCheckResult = - locationHeaderCheck(request.headers, objectKey, bucketName); + const locCheckResult = locationHeaderCheck(request.headers, objectKey, bucketName); if (locCheckResult instanceof Error) { log.trace('invalid location constraint to get from', { location: request.headers['x-amz-location-constraint'], @@ -73,294 +70,269 @@ function objectGet(authInfo, request, returnTagCount, log, callback) { returnTagCount, }; - return standardMetadataValidateBucketAndObj(mdValParams, request.actionImplicitDenies, log, - (err, bucket, objMD) => updateEncryption(err, bucket, objMD, objectKey, log, {}, - (err, bucket, objMD) => { - const corsHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); - if (err) { - log.debug('error processing request', { - error: err, - method: 'metadataValidateBucketAndObj', - }); - monitoring.promMetrics( - 'GET', bucketName, err.code, 'getObject'); - return callback(err, null, corsHeaders); - } - if (!objMD) { - const err = versionId ? errors.NoSuchVersion : errors.NoSuchKey; - monitoring.promMetrics( - 'GET', bucketName, err.code, 'getObject'); - return callback(err, null, corsHeaders); - } - const verCfg = bucket.getVersioningConfiguration(); - // check if object data is in a cold storage - const coldErr = verifyColdObjectAvailable(objMD); - if (coldErr) { - monitoring.promMetrics( - 'GET', bucketName, coldErr.code, 'getObject'); - return callback(coldErr, null, corsHeaders); - } - if (objMD.isDeleteMarker) { - const responseMetaHeaders = Object.assign({}, - { 'x-amz-delete-marker': true }, corsHeaders); - if (!versionId) { - monitoring.promMetrics( - 'GET', bucketName, 404, 'getObject'); - return callback(errors.NoSuchKey, null, responseMetaHeaders); + return standardMetadataValidateBucketAndObj(mdValParams, request.actionImplicitDenies, log, (err, bucket, objMD) => + updateEncryption(err, bucket, objMD, objectKey, log, {}, (err, bucket, objMD) => { + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + if (err) { + log.debug('error processing request', { + error: err, + method: 'metadataValidateBucketAndObj', + }); + monitoring.promMetrics('GET', bucketName, err.code, 'getObject'); + return callback(err, null, corsHeaders); } - // return MethodNotAllowed if requesting a specific - // version that has a delete marker - responseMetaHeaders['x-amz-version-id'] = - getVersionIdResHeader(verCfg, objMD); - monitoring.promMetrics( - 'GET', bucketName, 405, 'getObject'); - return callback(errors.MethodNotAllowed, null, - responseMetaHeaders); - } - const headerValResult = validateHeaders(request.headers, - objMD['last-modified'], objMD['content-md5']); - if (headerValResult.error) { - return callback(headerValResult.error, null, corsHeaders); - } - const responseMetaHeaders = collectResponseHeaders(objMD, - corsHeaders, verCfg, - returnTagCount && objMD.returnTagCount); // IAM and Bucket policy should both authorize tagging. - - setExpirationHeaders(responseMetaHeaders, { - lifecycleConfig: bucket.getLifecycleConfiguration(), - objectParams: { - key: objectKey, - tags: objMD.tags, - date: objMD['last-modified'], - }, - isVersionedReq: !!versionId, - }); - - const objLength = (objMD.location === null ? - 0 : parseInt(objMD['content-length'], 10)); - // Store full object size for server access logs - if (request.serverAccessLog) { - // eslint-disable-next-line no-param-reassign - request.serverAccessLog.objectSize = objLength; - } - let byteRange; - const streamingParams = {}; - if (request.headers.range) { - const { range, error } = parseRange(request.headers.range, - objLength); - if (error) { - monitoring.promMetrics( - 'GET', bucketName, 400, 'getObject'); - return callback(error, null, corsHeaders); + if (!objMD) { + const err = versionId ? errors.NoSuchVersion : errors.NoSuchKey; + monitoring.promMetrics('GET', bucketName, err.code, 'getObject'); + return callback(err, null, corsHeaders); } - responseMetaHeaders['Accept-Ranges'] = 'bytes'; - if (range) { - byteRange = range; - // End of range should be included so + 1 - responseMetaHeaders['Content-Length'] = - range[1] - range[0] + 1; - responseMetaHeaders['Content-Range'] = - `bytes ${range[0]}-${range[1]}/${objLength}`; - streamingParams.rangeStart = (range[0] || typeof range[0] === 'number') ? - range[0].toString() : undefined; - streamingParams.rangeEnd = range[1] ? - range[1].toString() : undefined; + const verCfg = bucket.getVersioningConfiguration(); + // check if object data is in a cold storage + const coldErr = verifyColdObjectAvailable(objMD); + if (coldErr) { + monitoring.promMetrics('GET', bucketName, coldErr.code, 'getObject'); + return callback(coldErr, null, corsHeaders); } - } - let dataLocator = null; - if (objMD.location !== null) { - // To provide for backwards compatibility before - // md-model-version 2, need to handle cases where - // objMD.location is just a string - dataLocator = Array.isArray(objMD.location) ? - objMD.location : [{ key: objMD.location }]; - - const repConf = bucket.getReplicationConfiguration(); - const prefReadLocation = repConf && repConf.preferredReadLocation; - const prefReadDataLocator = checkReadLocation(config, - prefReadLocation, objectKey, bucketName); - const targetLocation = locCheckResult || prefReadDataLocator || - null; - - if (targetLocation && - targetLocation.location !== objMD.dataStoreName) { - const repBackendResult = getReplicationBackendDataLocator( - targetLocation, objMD.replicationInfo); - if (repBackendResult.error) { - log.error('Error with location constraint header', { - bucketName, objectKey, versionId, - error: repBackendResult.error, - status: repBackendResult.status, - }); - return callback(repBackendResult.error, null, corsHeaders); - } - const targetDataLocator = repBackendResult.dataLocator; - if (targetDataLocator) { - dataLocator = targetDataLocator; - } else { - log.debug('using source location as preferred read ' + - 'is unavailable', { - bucketName, objectKey, versionId, - reason: repBackendResult.reason, - }); + if (objMD.isDeleteMarker) { + const responseMetaHeaders = Object.assign({}, { 'x-amz-delete-marker': true }, corsHeaders); + if (!versionId) { + monitoring.promMetrics('GET', bucketName, 404, 'getObject'); + return callback(errors.NoSuchKey, null, responseMetaHeaders); } + // return MethodNotAllowed if requesting a specific + // version that has a delete marker + responseMetaHeaders['x-amz-version-id'] = getVersionIdResHeader(verCfg, objMD); + monitoring.promMetrics('GET', bucketName, 405, 'getObject'); + return callback(errors.MethodNotAllowed, null, responseMetaHeaders); } - // if the data backend is azure, there will only ever be at - // most one item in the dataLocator array - if (dataLocator[0] && dataLocator[0].dataStoreType === 'azure') { - dataLocator[0].azureStreamingOptions = streamingParams; + const headerValResult = validateHeaders(request.headers, objMD['last-modified'], objMD['content-md5']); + if (headerValResult.error) { + return callback(headerValResult.error, null, corsHeaders); } + const responseMetaHeaders = collectResponseHeaders( + objMD, + corsHeaders, + verCfg, + returnTagCount && objMD.returnTagCount, + ); // IAM and Bucket policy should both authorize tagging. - let partNumber = null; - if (request.query && request.query.partNumber !== undefined) { - if (byteRange) { - const error = errorInstances.InvalidRequest - .customizeDescription('Cannot specify both Range ' + - 'header and partNumber query parameter.'); - monitoring.promMetrics( - 'GET', bucketName, 400, 'getObject'); - return callback(error, null, corsHeaders); - } - partNumber = Number.parseInt(request.query.partNumber, 10); - if (Number.isNaN(partNumber)) { - const error = errorInstances.InvalidArgument - .customizeDescription('Part number must be a number.'); - monitoring.promMetrics( - 'GET', bucketName, 400, 'getObject'); - return callback(error, null, corsHeaders); - } - if (partNumber < 1 || partNumber > 10000) { - const error = errorInstances.InvalidArgument - .customizeDescription('Part number must be an ' + - 'integer between 1 and 10000, inclusive.'); - monitoring.promMetrics( - 'GET', bucketName, 400, 'getObject'); + setExpirationHeaders(responseMetaHeaders, { + lifecycleConfig: bucket.getLifecycleConfiguration(), + objectParams: { + key: objectKey, + tags: objMD.tags, + date: objMD['last-modified'], + }, + isVersionedReq: !!versionId, + }); + + const objLength = objMD.location === null ? 0 : parseInt(objMD['content-length'], 10); + // Store full object size for server access logs + if (request.serverAccessLog) { + // eslint-disable-next-line no-param-reassign + request.serverAccessLog.objectSize = objLength; + } + let byteRange; + const streamingParams = {}; + if (request.headers.range) { + const { range, error } = parseRange(request.headers.range, objLength); + if (error) { + monitoring.promMetrics('GET', bucketName, 400, 'getObject'); return callback(error, null, corsHeaders); } - } - // If have a data model before version 2, cannot support - // get range for objects with multiple parts - if (byteRange && dataLocator.length > 1 && - dataLocator[0].start === undefined) { - monitoring.promMetrics( - 'GET', bucketName, 501, 'getObject'); - return callback(errors.NotImplemented, null, corsHeaders); - } - if (objMD['x-amz-server-side-encryption']) { - for (let i = 0; i < dataLocator.length; i++) { - dataLocator[i].masterKeyId = - objMD['x-amz-server-side-encryption-aws-kms-key-id']; - dataLocator[i].algorithm = - objMD['x-amz-server-side-encryption']; + responseMetaHeaders['Accept-Ranges'] = 'bytes'; + if (range) { + byteRange = range; + // End of range should be included so + 1 + responseMetaHeaders['Content-Length'] = range[1] - range[0] + 1; + responseMetaHeaders['Content-Range'] = `bytes ${range[0]}-${range[1]}/${objLength}`; + streamingParams.rangeStart = + range[0] || typeof range[0] === 'number' ? range[0].toString() : undefined; + streamingParams.rangeEnd = range[1] ? range[1].toString() : undefined; } } - if (partNumber) { - const locations = []; - let locationPartNumber; - for (let i = 0; i < objMD.location.length; i++) { - const { dataStoreETag } = objMD.location[i]; + let dataLocator = null; + if (objMD.location !== null) { + // To provide for backwards compatibility before + // md-model-version 2, need to handle cases where + // objMD.location is just a string + dataLocator = Array.isArray(objMD.location) ? objMD.location : [{ key: objMD.location }]; + + const repConf = bucket.getReplicationConfiguration(); + const prefReadLocation = repConf && repConf.preferredReadLocation; + const prefReadDataLocator = checkReadLocation(config, prefReadLocation, objectKey, bucketName); + const targetLocation = locCheckResult || prefReadDataLocator || null; - if (dataStoreETag) { - locationPartNumber = - Number.parseInt(dataStoreETag.split(':')[0], 10); + if (targetLocation && targetLocation.location !== objMD.dataStoreName) { + const repBackendResult = getReplicationBackendDataLocator(targetLocation, objMD.replicationInfo); + if (repBackendResult.error) { + log.error('Error with location constraint header', { + bucketName, + objectKey, + versionId, + error: repBackendResult.error, + status: repBackendResult.status, + }); + return callback(repBackendResult.error, null, corsHeaders); + } + const targetDataLocator = repBackendResult.dataLocator; + if (targetDataLocator) { + dataLocator = targetDataLocator; } else { - /** - * Location objects prior to GA7.1 do not include the - * dataStoreETag field so we cannot find the part range, - * the objects are treated as if they only have 1 part - */ - locationPartNumber = 1; + log.debug('using source location as preferred read ' + 'is unavailable', { + bucketName, + objectKey, + versionId, + reason: repBackendResult.reason, + }); } + } + // if the data backend is azure, there will only ever be at + // most one item in the dataLocator array + if (dataLocator[0] && dataLocator[0].dataStoreType === 'azure') { + dataLocator[0].azureStreamingOptions = streamingParams; + } - // Get all parts that belong to the requested part number - if (partNumber === locationPartNumber) { - locations.push(objMD.location[i]); - } else if (locationPartNumber > partNumber) { - break; + let partNumber = null; + if (request.query && request.query.partNumber !== undefined) { + if (byteRange) { + const error = errorInstances.InvalidRequest.customizeDescription( + 'Cannot specify both Range ' + 'header and partNumber query parameter.', + ); + monitoring.promMetrics('GET', bucketName, 400, 'getObject'); + return callback(error, null, corsHeaders); + } + partNumber = Number.parseInt(request.query.partNumber, 10); + if (Number.isNaN(partNumber)) { + const error = errorInstances.InvalidArgument.customizeDescription( + 'Part number must be a number.', + ); + monitoring.promMetrics('GET', bucketName, 400, 'getObject'); + return callback(error, null, corsHeaders); + } + if (partNumber < 1 || partNumber > 10000) { + const error = errorInstances.InvalidArgument.customizeDescription( + 'Part number must be an ' + 'integer between 1 and 10000, inclusive.', + ); + monitoring.promMetrics('GET', bucketName, 400, 'getObject'); + return callback(error, null, corsHeaders); } } - if (locations.length === 0) { - monitoring.promMetrics( - 'GET', bucketName, 400, 'getObject'); - return callback(errors.InvalidPartNumber, null, - corsHeaders); + // If have a data model before version 2, cannot support + // get range for objects with multiple parts + if (byteRange && dataLocator.length > 1 && dataLocator[0].start === undefined) { + monitoring.promMetrics('GET', bucketName, 501, 'getObject'); + return callback(errors.NotImplemented, null, corsHeaders); } - const { start } = locations[0]; - const endLocation = locations[locations.length - 1]; - const end = endLocation.start + endLocation.size - 1; - responseMetaHeaders['Content-Length'] = end - start + 1; - const partByteRange = [start, end]; - dataLocator = setPartRanges(dataLocator, partByteRange); - const partsCount = getPartCountFromMd5(objMD); - if (partsCount) { - responseMetaHeaders['x-amz-mp-parts-count'] = - partsCount; + if (objMD['x-amz-server-side-encryption']) { + for (let i = 0; i < dataLocator.length; i++) { + dataLocator[i].masterKeyId = objMD['x-amz-server-side-encryption-aws-kms-key-id']; + dataLocator[i].algorithm = objMD['x-amz-server-side-encryption']; + } } - } else { - dataLocator = setPartRanges(dataLocator, byteRange); - } - } - // Check KMS Key access and usability before checking data - // diff with AWS: for empty object (no dataLocator) KMS not checked - return async.each(dataLocator || [], - (objectGetInfo, next) => { - if (!objectGetInfo.cipheredDataKey) { - return next(); + if (partNumber) { + const locations = []; + let locationPartNumber; + for (let i = 0; i < objMD.location.length; i++) { + const { dataStoreETag } = objMD.location[i]; + + if (dataStoreETag) { + locationPartNumber = Number.parseInt(dataStoreETag.split(':')[0], 10); + } else { + /** + * Location objects prior to GA7.1 do not include the + * dataStoreETag field so we cannot find the part range, + * the objects are treated as if they only have 1 part + */ + locationPartNumber = 1; + } + + // Get all parts that belong to the requested part number + if (partNumber === locationPartNumber) { + locations.push(objMD.location[i]); + } else if (locationPartNumber > partNumber) { + break; + } + } + if (locations.length === 0) { + monitoring.promMetrics('GET', bucketName, 400, 'getObject'); + return callback(errors.InvalidPartNumber, null, corsHeaders); + } + const { start } = locations[0]; + const endLocation = locations[locations.length - 1]; + const end = endLocation.start + endLocation.size - 1; + responseMetaHeaders['Content-Length'] = end - start + 1; + const partByteRange = [start, end]; + dataLocator = setPartRanges(dataLocator, partByteRange); + const partsCount = getPartCountFromMd5(objMD); + if (partsCount) { + responseMetaHeaders['x-amz-mp-parts-count'] = partsCount; + } + } else { + dataLocator = setPartRanges(dataLocator, byteRange); } - const serverSideEncryption = { - cryptoScheme: objectGetInfo.cryptoScheme, - masterKeyId: objectGetInfo.masterKeyId, - cipheredDataKey: Buffer.from( - objectGetInfo.cipheredDataKey, 'base64'), - }; - const offset = objectGetInfo.range ? objectGetInfo.range[0] : 0; - return kms.createDecipherBundle(serverSideEncryption, - offset, log, (err, decipherBundle) => { + } + // Check KMS Key access and usability before checking data + // diff with AWS: for empty object (no dataLocator) KMS not checked + return async.each( + dataLocator || [], + (objectGetInfo, next) => { + if (!objectGetInfo.cipheredDataKey) { + return next(); + } + const serverSideEncryption = { + cryptoScheme: objectGetInfo.cryptoScheme, + masterKeyId: objectGetInfo.masterKeyId, + cipheredDataKey: Buffer.from(objectGetInfo.cipheredDataKey, 'base64'), + }; + const offset = objectGetInfo.range ? objectGetInfo.range[0] : 0; + return kms.createDecipherBundle(serverSideEncryption, offset, log, (err, decipherBundle) => { if (err) { - log.error('cannot get decipher bundle from kms', - { method: 'objectGet' }); + log.error('cannot get decipher bundle from kms', { method: 'objectGet' }); return next(err); } // eslint-disable-next-line no-param-reassign objectGetInfo.decipherStream = decipherBundle.decipher; return next(); }); - }, - err => { - if (err) { - monitoring.promMetrics( - 'GET', bucketName, err.code, 'getObject'); - return callback(err); - } - - return data.head(dataLocator, log, err => { + }, + err => { if (err) { - if (!err.is.LocationNotFound) { - log.error('error from external backend checking for ' + - 'object existence', { error: err }); - } - monitoring.promMetrics( - 'GET', bucketName, err.code, 'getObject'); + monitoring.promMetrics('GET', bucketName, err.code, 'getObject'); return callback(err); } - pushMetric('getObject', log, { - authInfo, - bucket: bucketName, - keys: [objectKey], - newByteLength: + + return data.head(dataLocator, log, err => { + if (err) { + if (!err.is.LocationNotFound) { + log.error('error from external backend checking for ' + 'object existence', { + error: err, + }); + } + monitoring.promMetrics('GET', bucketName, err.code, 'getObject'); + return callback(err); + } + pushMetric('getObject', log, { + authInfo, + bucket: bucketName, + keys: [objectKey], + newByteLength: Number.parseInt(responseMetaHeaders['Content-Length'], 10), + versionId: objMD.versionId, + location: objMD.dataStoreName, + }); + monitoring.promMetrics( + 'GET', + bucketName, + '200', + 'getObject', Number.parseInt(responseMetaHeaders['Content-Length'], 10), - versionId: objMD.versionId, - location: objMD.dataStoreName, + ); + return callback(null, dataLocator, responseMetaHeaders, byteRange); }); - monitoring.promMetrics('GET', bucketName, '200', 'getObject', - Number.parseInt(responseMetaHeaders['Content-Length'], 10)); - return callback(null, dataLocator, responseMetaHeaders, - byteRange); - }); - } - ); - })); + }, + ); + }), + ); } module.exports = objectGet; diff --git a/lib/api/objectGetACL.js b/lib/api/objectGetACL.js index 5f4045bbc0..030ebb8e22 100644 --- a/lib/api/objectGetACL.js +++ b/lib/api/objectGetACL.js @@ -4,8 +4,7 @@ const { errors } = require('arsenal'); const aclUtils = require('../utilities/aclUtils'); const collectCorsHeaders = require('../utilities/collectCorsHeaders'); const { pushMetric } = require('../utapi/utilities'); -const { decodeVersionId, getVersionIdResHeader } - = require('./apiUtils/object/versioning'); +const { decodeVersionId, getVersionIdResHeader } = require('./apiUtils/object/versioning'); const vault = require('../auth/vault'); const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUtils'); const monitoring = require('../utilities/monitoringHandler'); @@ -72,128 +71,123 @@ function objectGetACL(authInfo, request, log, callback) { }, }; - return async.waterfall([ - function validateBucketAndObj(next) { - return standardMetadataValidateBucketAndObj(metadataValParams, request.actionImplicitDenies, log, - (err, bucket, objectMD) => { - if (err) { - log.trace('request authorization failed', - { method: 'objectGetACL', error: err }); - return next(err); - } - if (!objectMD) { - const err = versionId ? errors.NoSuchVersion : - errors.NoSuchKey; - log.trace('error processing request', - { method: 'objectGetACL', error: err }); - return next(err, bucket); - } - if (objectMD.isDeleteMarker) { - if (versionId) { - log.trace('requested version is delete marker', - { method: 'objectGetACL' }); + return async.waterfall( + [ + function validateBucketAndObj(next) { + return standardMetadataValidateBucketAndObj( + metadataValParams, + request.actionImplicitDenies, + log, + (err, bucket, objectMD) => { + if (err) { + log.trace('request authorization failed', { method: 'objectGetACL', error: err }); + return next(err); + } + if (!objectMD) { + const err = versionId ? errors.NoSuchVersion : errors.NoSuchKey; + log.trace('error processing request', { method: 'objectGetACL', error: err }); + return next(err, bucket); + } + if (objectMD.isDeleteMarker) { + if (versionId) { + log.trace('requested version is delete marker', { method: 'objectGetACL' }); + // FIXME we should return a `x-amz-delete-marker: true` header, + // see S3C-7592 + return next(errors.MethodNotAllowed); + } + log.trace('most recent version is delete marker', { method: 'objectGetACL' }); // FIXME we should return a `x-amz-delete-marker: true` header, // see S3C-7592 - return next(errors.MethodNotAllowed); + return next(errors.NoSuchKey); } - log.trace('most recent version is delete marker', - { method: 'objectGetACL' }); - // FIXME we should return a `x-amz-delete-marker: true` header, - // see S3C-7592 - return next(errors.NoSuchKey); + return next(null, bucket, objectMD); + }, + ); + }, + function gatherACLs(bucket, objectMD, next) { + const verCfg = bucket.getVersioningConfiguration(); + const resVersionId = getVersionIdResHeader(verCfg, objectMD); + const objectACL = objectMD.acl; + grantInfo.ownerInfo.ID = objectMD['owner-id']; + grantInfo.ownerInfo.displayName = objectMD['owner-display-name']; + // Object owner always has full control + const ownerGrant = { + ID: objectMD['owner-id'], + displayName: objectMD['owner-display-name'], + permission: 'FULL_CONTROL', + }; + if (objectACL.Canned !== '') { + /** + * If bucket owner and object owner are different + * need to send info about bucket owner from bucket + * metadata to handleCannedGrant function + */ + let cannedGrants; + if (bucket.getOwner() !== objectMD['owner-id']) { + cannedGrants = aclUtils.handleCannedGrant(objectACL.Canned, ownerGrant, bucket); + } else { + cannedGrants = aclUtils.handleCannedGrant(objectACL.Canned, ownerGrant); } - return next(null, bucket, objectMD); - }); - }, - function gatherACLs(bucket, objectMD, next) { - const verCfg = bucket.getVersioningConfiguration(); - const resVersionId = getVersionIdResHeader(verCfg, objectMD); - const objectACL = objectMD.acl; - grantInfo.ownerInfo.ID = objectMD['owner-id']; - grantInfo.ownerInfo.displayName = objectMD['owner-display-name']; - // Object owner always has full control - const ownerGrant = { - ID: objectMD['owner-id'], - displayName: objectMD['owner-display-name'], - permission: 'FULL_CONTROL', - }; - if (objectACL.Canned !== '') { - /** - * If bucket owner and object owner are different - * need to send info about bucket owner from bucket - * metadata to handleCannedGrant function - */ - let cannedGrants; - if (bucket.getOwner() !== objectMD['owner-id']) { - cannedGrants = aclUtils.handleCannedGrant( - objectACL.Canned, ownerGrant, bucket); - } else { - cannedGrants = aclUtils.handleCannedGrant( - objectACL.Canned, ownerGrant); + grantInfo.grants = grantInfo.grants.concat(cannedGrants); + const xml = aclUtils.convertToXml(grantInfo); + return next(null, bucket, xml, resVersionId); } - grantInfo.grants = grantInfo.grants.concat(cannedGrants); - const xml = aclUtils.convertToXml(grantInfo); - return next(null, bucket, xml, resVersionId); - } - /** - * Build array of all canonicalIDs used in ACLs so duplicates - * will be retained (e.g. if an account has both read and write - * privileges, want to display both and not lose the duplicate - * when receive one dictionary entry back from Vault) - */ - const canonicalIDs = aclUtils.getCanonicalIDs(objectACL); - // Build array with grants by URI - const uriGrantInfo = aclUtils.getUriGrantInfo(objectACL); + /** + * Build array of all canonicalIDs used in ACLs so duplicates + * will be retained (e.g. if an account has both read and write + * privileges, want to display both and not lose the duplicate + * when receive one dictionary entry back from Vault) + */ + const canonicalIDs = aclUtils.getCanonicalIDs(objectACL); + // Build array with grants by URI + const uriGrantInfo = aclUtils.getUriGrantInfo(objectACL); - if (canonicalIDs.length === 0) { + if (canonicalIDs.length === 0) { + /** + * If no acl's set by account canonicalID, just add URI + * grants (if any) and return + */ + grantInfo.grants = grantInfo.grants.concat(uriGrantInfo); + const xml = aclUtils.convertToXml(grantInfo); + return next(null, bucket, xml, resVersionId); + } /** - * If no acl's set by account canonicalID, just add URI - * grants (if any) and return - */ - grantInfo.grants = grantInfo.grants.concat(uriGrantInfo); - const xml = aclUtils.convertToXml(grantInfo); - return next(null, bucket, xml, resVersionId); + * If acl's set by account canonicalID, + * get emails from Vault to serve + * as display names + */ + return vault.getEmailAddresses(canonicalIDs, log, (err, emails) => { + if (err) { + log.trace('error processing request', { method: 'objectGetACL', error: err }); + return next(err, bucket); + } + const individualGrants = aclUtils.getIndividualGrants(objectACL, canonicalIDs, emails); + // Add to grantInfo any individual grants and grants by uri + grantInfo.grants = grantInfo.grants.concat(individualGrants).concat(uriGrantInfo); + // parse info about accounts and owner info to convert to xml + const xml = aclUtils.convertToXml(grantInfo); + return next(null, bucket, xml, resVersionId); + }); + }, + ], + (err, bucket, xml, resVersionId) => { + const resHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + if (err) { + monitoring.promMetrics('GET', bucketName, err.code, 'getObjectAcl'); + return callback(err, null, resHeaders); } - /** - * If acl's set by account canonicalID, - * get emails from Vault to serve - * as display names - */ - return vault.getEmailAddresses(canonicalIDs, log, (err, emails) => { - if (err) { - log.trace('error processing request', - { method: 'objectGetACL', error: err }); - return next(err, bucket); - } - const individualGrants = aclUtils.getIndividualGrants(objectACL, - canonicalIDs, emails); - // Add to grantInfo any individual grants and grants by uri - grantInfo.grants = grantInfo.grants - .concat(individualGrants).concat(uriGrantInfo); - // parse info about accounts and owner info to convert to xml - const xml = aclUtils.convertToXml(grantInfo); - return next(null, bucket, xml, resVersionId); + pushMetric('getObjectAcl', log, { + authInfo, + bucket: bucketName, + keys: [objectKey], + versionId: resVersionId, + location: bucket ? bucket.getLocationConstraint() : undefined, }); + monitoring.promMetrics('GET', bucketName, '200', 'getObjectAcl'); + resHeaders['x-amz-version-id'] = resVersionId; + return callback(null, xml, resHeaders); }, - ], (err, bucket, xml, resVersionId) => { - const resHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); - if (err) { - monitoring.promMetrics( - 'GET', bucketName, err.code, 'getObjectAcl'); - return callback(err, null, resHeaders); - } - pushMetric('getObjectAcl', log, { - authInfo, - bucket: bucketName, - keys: [objectKey], - versionId: resVersionId, - location: bucket ? bucket.getLocationConstraint() : undefined, - }); - monitoring.promMetrics('GET', bucketName, '200', 'getObjectAcl'); - resHeaders['x-amz-version-id'] = resVersionId; - return callback(null, xml, resHeaders); - }); + ); } module.exports = objectGetACL; diff --git a/lib/api/objectGetAttributes.js b/lib/api/objectGetAttributes.js index 1610d078ec..93e9b80b11 100644 --- a/lib/api/objectGetAttributes.js +++ b/lib/api/objectGetAttributes.js @@ -20,10 +20,7 @@ const OBJECT_GET_ATTRIBUTES = 'objectGetAttributes'; * @returns {string} XML response */ function buildXmlResponse(objMD, requestedAttrs) { - const xml = [ - '', - '', - ]; + const xml = ['', '']; const userMetadata = getUserMetadata(objMD); buildAttributesXml(objMD, userMetadata, requestedAttrs, xml); @@ -32,7 +29,6 @@ function buildXmlResponse(objMD, requestedAttrs) { return xml.join(''); } - /** * getUserMetadata - Retrieves all object user metadata * @param {object} objMD - object metadata diff --git a/lib/api/objectGetLegalHold.js b/lib/api/objectGetLegalHold.js index 8940c2ba8c..e815493333 100644 --- a/lib/api/objectGetLegalHold.js +++ b/lib/api/objectGetLegalHold.js @@ -1,8 +1,7 @@ const { promisify } = require('util'); const { errors, errorInstances, s3middleware } = require('arsenal'); -const { decodeVersionId, getVersionIdResHeader } - = require('./apiUtils/object/versioning'); +const { decodeVersionId, getVersionIdResHeader } = require('./apiUtils/object/versioning'); const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUtils'); const { pushMetric } = require('../utapi/utilities'); diff --git a/lib/api/objectGetRetention.js b/lib/api/objectGetRetention.js index 3ac4c19c98..6756be4026 100644 --- a/lib/api/objectGetRetention.js +++ b/lib/api/objectGetRetention.js @@ -1,8 +1,7 @@ const async = require('async'); const { errors, errorInstances, s3middleware } = require('arsenal'); -const { decodeVersionId, getVersionIdResHeader } - = require('./apiUtils/object/versioning'); +const { decodeVersionId, getVersionIdResHeader } = require('./apiUtils/object/versioning'); const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUtils'); const { pushMetric } = require('../utapi/utilities'); @@ -44,71 +43,73 @@ function objectGetRetention(authInfo, request, log, callback) { request, }; - return async.waterfall([ - next => standardMetadataValidateBucketAndObj(metadataValParams, request.actionImplicitDenies, log, - (err, bucket, objectMD) => { - if (err) { - log.trace('request authorization failed', - { method: 'objectGetRetention', error: err }); - return next(err); + return async.waterfall( + [ + next => + standardMetadataValidateBucketAndObj( + metadataValParams, + request.actionImplicitDenies, + log, + (err, bucket, objectMD) => { + if (err) { + log.trace('request authorization failed', { method: 'objectGetRetention', error: err }); + return next(err); + } + if (!objectMD) { + const err = reqVersionId ? errors.NoSuchVersion : errors.NoSuchKey; + log.trace('error no object metadata found', { method: 'objectGetRetention', error: err }); + return next(err, bucket); + } + if (objectMD.isDeleteMarker) { + if (reqVersionId) { + log.trace('requested version is delete marker', { method: 'objectGetRetention' }); + // FIXME we should return a `x-amz-delete-marker: true` header, + // see S3C-7592 + return next(errors.MethodNotAllowed); + } + log.trace('most recent version is delete marker', { method: 'objectGetRetention' }); + // FIXME we should return a `x-amz-delete-marker: true` header, + // see S3C-7592 + return next(errors.NoSuchKey); + } + if (!bucket.isObjectLockEnabled()) { + log.trace('object lock not enabled on bucket', { method: 'objectGetRetention' }); + return next( + errorInstances.InvalidRequest.customizeDescription( + 'Bucket is missing Object Lock Configuration', + ), + ); + } + return next(null, bucket, objectMD); + }, + ), + (bucket, objectMD, next) => { + const { retentionMode, retentionDate } = objectMD; + if (!retentionMode || !retentionDate) { + return next(errors.NoSuchObjectLockConfiguration); } - if (!objectMD) { - const err = reqVersionId ? errors.NoSuchVersion : - errors.NoSuchKey; - log.trace('error no object metadata found', - { method: 'objectGetRetention', error: err }); - return next(err, bucket); - } - if (objectMD.isDeleteMarker) { - if (reqVersionId) { - log.trace('requested version is delete marker', - { method: 'objectGetRetention' }); - // FIXME we should return a `x-amz-delete-marker: true` header, - // see S3C-7592 - return next(errors.MethodNotAllowed); - } - log.trace('most recent version is delete marker', - { method: 'objectGetRetention' }); - // FIXME we should return a `x-amz-delete-marker: true` header, - // see S3C-7592 - return next(errors.NoSuchKey); - } - if (!bucket.isObjectLockEnabled()) { - log.trace('object lock not enabled on bucket', - { method: 'objectGetRetention' }); - return next(errorInstances.InvalidRequest.customizeDescription( - 'Bucket is missing Object Lock Configuration')); - } - return next(null, bucket, objectMD); - }), - (bucket, objectMD, next) => { - const { retentionMode, retentionDate } = objectMD; - if (!retentionMode || !retentionDate) { - return next(errors.NoSuchObjectLockConfiguration); + const xml = convertToXml(retentionMode, retentionDate); + return next(null, bucket, xml, objectMD); + }, + ], + (err, bucket, xml, objectMD) => { + const additionalResHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + if (err) { + log.trace('error processing request', { error: err, method: 'objectGetRetention' }); + } else { + pushMetric('getObjectRetention', log, { + authInfo, + bucket: bucketName, + keys: [objectKey], + versionId: objectMD ? objectMD.versionId : undefined, + location: objectMD ? objectMD.dataStoreName : undefined, + }); + const verCfg = bucket.getVersioningConfiguration(); + additionalResHeaders['x-amz-version-id'] = getVersionIdResHeader(verCfg, objectMD); } - const xml = convertToXml(retentionMode, retentionDate); - return next(null, bucket, xml, objectMD); + return callback(err, xml, additionalResHeaders); }, - ], (err, bucket, xml, objectMD) => { - const additionalResHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); - if (err) { - log.trace('error processing request', { error: err, - method: 'objectGetRetention' }); - } else { - pushMetric('getObjectRetention', log, { - authInfo, - bucket: bucketName, - keys: [objectKey], - versionId: objectMD ? objectMD.versionId : undefined, - location: objectMD ? objectMD.dataStoreName : undefined, - }); - const verCfg = bucket.getVersioningConfiguration(); - additionalResHeaders['x-amz-version-id'] = - getVersionIdResHeader(verCfg, objectMD); - } - return callback(err, xml, additionalResHeaders); - }); + ); } module.exports = objectGetRetention; diff --git a/lib/api/objectGetTagging.js b/lib/api/objectGetTagging.js index 9d52b528a7..4a743b24fc 100644 --- a/lib/api/objectGetTagging.js +++ b/lib/api/objectGetTagging.js @@ -1,8 +1,7 @@ const async = require('async'); const { errors, s3middleware } = require('arsenal'); -const { decodeVersionId, getVersionIdResHeader } - = require('./apiUtils/object/versioning'); +const { decodeVersionId, getVersionIdResHeader } = require('./apiUtils/object/versioning'); const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUtils'); const { pushMetric } = require('../utapi/utilities'); @@ -45,66 +44,64 @@ function objectGetTagging(authInfo, request, log, callback) { request, }; - return async.waterfall([ - next => standardMetadataValidateBucketAndObj(metadataValParams, request.actionImplicitDenies, log, - (err, bucket, objectMD) => { - if (err) { - log.trace('request authorization failed', - { method: 'objectGetTagging', error: err }); - return next(err); - } - if (!objectMD) { - const err = reqVersionId ? errors.NoSuchVersion : - errors.NoSuchKey; - log.trace('error no object metadata found', - { method: 'objectGetTagging', error: err }); - return next(err, bucket); - } - if (objectMD.isDeleteMarker) { - if (reqVersionId) { - log.trace('requested version is delete marker', - { method: 'objectGetTagging' }); - return next(errors.MethodNotAllowed); - // FIXME we should return a `x-amz-delete-marker: true` header, - // see S3C-7592 - } - log.trace('most recent version is delete marker', - { method: 'objectGetTagging' }); - // FIXME we should return a `x-amz-delete-marker: true` header, - // see S3C-7592 - return next(errors.NoSuchKey); - } - return next(null, bucket, objectMD); - }), - (bucket, objectMD, next) => { - const tags = objectMD.tags; - const xml = convertToXml(tags); - next(null, bucket, xml, objectMD); + return async.waterfall( + [ + next => + standardMetadataValidateBucketAndObj( + metadataValParams, + request.actionImplicitDenies, + log, + (err, bucket, objectMD) => { + if (err) { + log.trace('request authorization failed', { method: 'objectGetTagging', error: err }); + return next(err); + } + if (!objectMD) { + const err = reqVersionId ? errors.NoSuchVersion : errors.NoSuchKey; + log.trace('error no object metadata found', { method: 'objectGetTagging', error: err }); + return next(err, bucket); + } + if (objectMD.isDeleteMarker) { + if (reqVersionId) { + log.trace('requested version is delete marker', { method: 'objectGetTagging' }); + return next(errors.MethodNotAllowed); + // FIXME we should return a `x-amz-delete-marker: true` header, + // see S3C-7592 + } + log.trace('most recent version is delete marker', { method: 'objectGetTagging' }); + // FIXME we should return a `x-amz-delete-marker: true` header, + // see S3C-7592 + return next(errors.NoSuchKey); + } + return next(null, bucket, objectMD); + }, + ), + (bucket, objectMD, next) => { + const tags = objectMD.tags; + const xml = convertToXml(tags); + next(null, bucket, xml, objectMD); + }, + ], + (err, bucket, xml, objectMD) => { + const additionalResHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + if (err) { + log.trace('error processing request', { error: err, method: 'objectGetTagging' }); + monitoring.promMetrics('GET', bucketName, err.code, 'getObjectTagging'); + } else { + pushMetric('getObjectTagging', log, { + authInfo, + bucket: bucketName, + keys: [objectKey], + versionId: objectMD ? objectMD.versionId : undefined, + location: objectMD ? objectMD.dataStoreName : undefined, + }); + monitoring.promMetrics('GET', bucketName, '200', 'getObjectTagging'); + const verCfg = bucket.getVersioningConfiguration(); + additionalResHeaders['x-amz-version-id'] = getVersionIdResHeader(verCfg, objectMD); + } + return callback(err, xml, additionalResHeaders); }, - ], (err, bucket, xml, objectMD) => { - const additionalResHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); - if (err) { - log.trace('error processing request', { error: err, - method: 'objectGetTagging' }); - monitoring.promMetrics( - 'GET', bucketName, err.code, 'getObjectTagging'); - } else { - pushMetric('getObjectTagging', log, { - authInfo, - bucket: bucketName, - keys: [objectKey], - versionId: objectMD ? objectMD.versionId : undefined, - location: objectMD ? objectMD.dataStoreName : undefined, - }); - monitoring.promMetrics( - 'GET', bucketName, '200', 'getObjectTagging'); - const verCfg = bucket.getVersioningConfiguration(); - additionalResHeaders['x-amz-version-id'] = - getVersionIdResHeader(verCfg, objectMD); - } - return callback(err, xml, additionalResHeaders); - }); + ); } module.exports = objectGetTagging; diff --git a/lib/api/objectHead.js b/lib/api/objectHead.js index 2d4c524f21..939780811a 100644 --- a/lib/api/objectHead.js +++ b/lib/api/objectHead.js @@ -8,8 +8,7 @@ const collectResponseHeaders = require('../utilities/collectResponseHeaders'); const { pushMetric } = require('../utapi/utilities'); const { getVersionIdResHeader } = require('./apiUtils/object/versioning'); const monitoring = require('../utilities/monitoringHandler'); -const { getPartNumber, getPartSize, getPartCountFromMd5 } = - require('./apiUtils/object/partInfo'); +const { getPartNumber, getPartSize, getPartCountFromMd5 } = require('./apiUtils/object/partInfo'); const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUtils'); const { maximumAllowedPartCount } = require('../../constants'); @@ -52,50 +51,40 @@ function objectHead(authInfo, request, log, callback) { request, }; - return standardMetadataValidateBucketAndObj(mdValParams, request.actionImplicitDenies, log, - (err, bucket, objMD) => updateEncryption(err, bucket, objMD, objectKey, log, {}, - (err, bucket, objMD) => { - const corsHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); + return standardMetadataValidateBucketAndObj(mdValParams, request.actionImplicitDenies, log, (err, bucket, objMD) => + updateEncryption(err, bucket, objMD, objectKey, log, {}, (err, bucket, objMD) => { + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); if (err) { log.debug('error validating request', { error: err, method: 'objectHead', }); - monitoring.promMetrics( - 'HEAD', bucketName, err.code, 'headObject'); + monitoring.promMetrics('HEAD', bucketName, err.code, 'headObject'); return callback(err, corsHeaders); } if (!objMD) { const err = versionId ? errors.NoSuchVersion : errors.NoSuchKey; - monitoring.promMetrics( - 'HEAD', bucketName, err.code, 'headObject'); + monitoring.promMetrics('HEAD', bucketName, err.code, 'headObject'); return callback(err, corsHeaders); } const verCfg = bucket.getVersioningConfiguration(); if (objMD.isDeleteMarker) { - const responseHeaders = Object.assign({}, - { 'x-amz-delete-marker': true }, corsHeaders); + const responseHeaders = Object.assign({}, { 'x-amz-delete-marker': true }, corsHeaders); if (!versionId) { - monitoring.promMetrics( - 'HEAD', bucketName, 404, 'headObject'); + monitoring.promMetrics('HEAD', bucketName, 404, 'headObject'); return callback(errors.NoSuchKey, responseHeaders); } // return MethodNotAllowed if requesting a specific // version that has a delete marker - responseHeaders['x-amz-version-id'] = - getVersionIdResHeader(verCfg, objMD); - monitoring.promMetrics( - 'HEAD', bucketName, 405, 'headObject'); + responseHeaders['x-amz-version-id'] = getVersionIdResHeader(verCfg, objMD); + monitoring.promMetrics('HEAD', bucketName, 405, 'headObject'); return callback(errors.MethodNotAllowed, responseHeaders); } - const headerValResult = validateHeaders(request.headers, - objMD['last-modified'], objMD['content-md5']); + const headerValResult = validateHeaders(request.headers, objMD['last-modified'], objMD['content-md5']); if (headerValResult.error) { return callback(headerValResult.error, corsHeaders); } - const responseHeaders = collectResponseHeaders(objMD, corsHeaders, - verCfg); + const responseHeaders = collectResponseHeaders(objMD, corsHeaders, verCfg); setExpirationHeaders(responseHeaders, { lifecycleConfig: bucket.getLifecycleConfiguration(), @@ -111,8 +100,7 @@ function objectHead(authInfo, request, log, callback) { Object.assign(responseHeaders, setArchiveInfoHeaders(objMD)); } - const objLength = (objMD.location === null ? - 0 : parseInt(objMD['content-length'], 10)); + const objLength = objMD.location === null ? 0 : parseInt(objMD['content-length'], 10); // Store full object size for server access logs if (request.serverAccessLog) { // eslint-disable-next-line no-param-reassign @@ -121,31 +109,27 @@ function objectHead(authInfo, request, log, callback) { let byteRange; if (request.headers.range) { - const { range, error } - = parseRange(request.headers.range, objLength); + const { range, error } = parseRange(request.headers.range, objLength); if (error) { return callback(error, corsHeaders); } responseHeaders['accept-ranges'] = 'bytes'; if (range) { byteRange = range; - responseHeaders['content-length'] = - range[1] - range[0] + 1; - responseHeaders['content-range'] = - `bytes ${range[0]}-${range[1]}/${objLength}`; + responseHeaders['content-length'] = range[1] - range[0] + 1; + responseHeaders['content-range'] = `bytes ${range[0]}-${range[1]}/${objLength}`; } } const partNumber = getPartNumber(request.query); if (partNumber !== undefined) { if (byteRange) { - const error = errorInstances.InvalidRequest - .customizeDescription('Cannot specify both Range ' + - 'header and partNumber query parameter.'); + const error = errorInstances.InvalidRequest.customizeDescription( + 'Cannot specify both Range ' + 'header and partNumber query parameter.', + ); return callback(error, corsHeaders); } if (Number.isNaN(partNumber)) { - const error = errorInstances.InvalidArgument - .customizeDescription('Part number must be a number.'); + const error = errorInstances.InvalidArgument.customizeDescription('Part number must be a number.'); return callback(error, corsHeaders); } if (partNumber < 1 || partNumber > maximumAllowedPartCount) { @@ -174,7 +158,8 @@ function objectHead(authInfo, request, log, callback) { }); monitoring.promMetrics('HEAD', bucketName, '200', 'headObject'); return callback(null, responseHeaders); - })); + }), + ); } module.exports = objectHead; diff --git a/lib/api/objectPut.js b/lib/api/objectPut.js index d34c3dadf6..7c89b098da 100644 --- a/lib/api/objectPut.js +++ b/lib/api/objectPut.js @@ -28,7 +28,8 @@ const versionIdUtils = versioning.VersionID; const { updateEncryption } = require('./apiUtils/bucket/updateEncryption'); const invalidSSEError = errorInstances.InvalidArgument.customizeDescription( - 'The encryption method specified is not supported'); + 'The encryption method specified is not supported', +); /** * PUT Object in the requested bucket. Steps include: @@ -69,25 +70,15 @@ function objectPut(authInfo, request, streamingV4Params, log, callback) { versionId = decodedVidResult; } - const { - bucketName, - headers, - method, - objectKey, - parsedContentLength, - query, - } = request; - if (headers['x-amz-storage-class'] && - !constants.validStorageClasses.includes(headers['x-amz-storage-class'])) { + const { bucketName, headers, method, objectKey, parsedContentLength, query } = request; + if (headers['x-amz-storage-class'] && !constants.validStorageClasses.includes(headers['x-amz-storage-class'])) { log.trace('invalid storage-class header'); - monitoring.promMetrics('PUT', request.bucketName, - errorInstances.InvalidStorageClass.code, 'putObject'); + monitoring.promMetrics('PUT', request.bucketName, errorInstances.InvalidStorageClass.code, 'putObject'); return callback(errors.InvalidStorageClass); } if (!aclUtils.checkGrantHeaderValidity(headers)) { log.trace('invalid acl header'); - monitoring.promMetrics('PUT', request.bucketName, 400, - 'putObject'); + monitoring.promMetrics('PUT', request.bucketName, 400, 'putObject'); return callback(errors.InvalidArgument); } const queryContainsVersionId = checkQueryVersionId(query); @@ -101,22 +92,19 @@ function objectPut(authInfo, request, streamingV4Params, log, callback) { } const size = request.parsedContentLength; - if (Number.parseInt(size, 10) > constants.maximumAllowedUploadSize - && !config.bypassMaxPutObjectSize) { - log.debug('Upload size exceeds maximum allowed for a single PUT', - { size }); + if (Number.parseInt(size, 10) > constants.maximumAllowedUploadSize && !config.bypassMaxPutObjectSize) { + log.debug('Upload size exceeds maximum allowed for a single PUT', { size }); return callback(errors.EntityTooLarge); } const requestType = request.apiMethods || 'objectPut'; - const valParams = { authInfo, bucketName, objectKey, versionId, - requestType, request, withVersionId: isPutVersion }; + const valParams = { authInfo, bucketName, objectKey, versionId, requestType, request, withVersionId: isPutVersion }; const canonicalID = authInfo.getCanonicalID(); if (hasNonPrintables(objectKey)) { - return callback(errorInstances.InvalidInput.customizeDescription( - 'object keys cannot contain non-printable characters', - )); + return callback( + errorInstances.InvalidInput.customizeDescription('object keys cannot contain non-printable characters'), + ); } const checksumHeaderErr = validateChecksumHeaders(headers); @@ -126,154 +114,174 @@ function objectPut(authInfo, request, streamingV4Params, log, callback) { log.trace('owner canonicalID to send to data', { canonicalID }); - return standardMetadataValidateBucketAndObj(valParams, request.actionImplicitDenies, log, - (err, bucket, objMD) => updateEncryption(err, bucket, objMD, objectKey, log, { skipObject: true }, - (err, bucket, objMD) => { - const responseHeaders = collectCorsHeaders(headers.origin, - method, bucket); - if (err) { - log.trace('error processing request', { - error: err, - method: 'metadataValidateBucketAndObj', - }); - monitoring.promMetrics('PUT', bucketName, err.code, 'putObject'); - return callback(err, responseHeaders); - } - if (bucket.hasDeletedFlag() && canonicalID !== bucket.getOwner()) { - log.trace('deleted flag on bucket and request ' + - 'from non-owner account'); - monitoring.promMetrics('PUT', bucketName, 404, 'putObject'); - return callback(errors.NoSuchBucket); - } - - if (isPutVersion) { - const error = validatePutVersionId(objMD, putVersionId, log); - if (error) { - return callback(error); + return standardMetadataValidateBucketAndObj(valParams, request.actionImplicitDenies, log, (err, bucket, objMD) => + updateEncryption(err, bucket, objMD, objectKey, log, { skipObject: true }, (err, bucket, objMD) => { + const responseHeaders = collectCorsHeaders(headers.origin, method, bucket); + if (err) { + log.trace('error processing request', { + error: err, + method: 'metadataValidateBucketAndObj', + }); + monitoring.promMetrics('PUT', bucketName, err.code, 'putObject'); + return callback(err, responseHeaders); + } + if (bucket.hasDeletedFlag() && canonicalID !== bucket.getOwner()) { + log.trace('deleted flag on bucket and request ' + 'from non-owner account'); + monitoring.promMetrics('PUT', bucketName, 404, 'putObject'); + return callback(errors.NoSuchBucket); } - } - return async.waterfall([ - function handleTransientOrDeleteBuckets(next) { - if (bucket.hasTransientFlag() || bucket.hasDeletedFlag()) { - return cleanUpBucket(bucket, canonicalID, log, next); + if (isPutVersion) { + const error = validatePutVersionId(objMD, putVersionId, log); + if (error) { + return callback(error); } - return next(); - }, - function getSSEConfig(next) { - return getObjectSSEConfiguration(headers, bucket, log, - (err, sseConfig) => { - if (err) { - log.error('error getting server side encryption config', { err }); - return next(invalidSSEError); + } + + return async.waterfall( + [ + function handleTransientOrDeleteBuckets(next) { + if (bucket.hasTransientFlag() || bucket.hasDeletedFlag()) { + return cleanUpBucket(bucket, canonicalID, log, next); } - return next(null, sseConfig); - } - ); - }, - function createCipherBundle(serverSideEncryptionConfig, next) { - if (serverSideEncryptionConfig) { - return kms.createCipherBundle( - serverSideEncryptionConfig, log, (err, cipherBundle) => { + return next(); + }, + function getSSEConfig(next) { + return getObjectSSEConfiguration(headers, bucket, log, (err, sseConfig) => { if (err) { - return next(err); + log.error('error getting server side encryption config', { err }); + return next(invalidSSEError); } - setSSEHeaders(responseHeaders, - cipherBundle.algorithm, - cipherBundle.configuredMasterKeyId || cipherBundle.masterKeyId); - return next(null, cipherBundle); + return next(null, sseConfig); }); - } - return next(null, null); - }, - function objectCreateAndStore(cipherBundle, next) { - const objectLockValidationError - = validateHeaders(bucket, headers, log); - if (objectLockValidationError) { - return next(objectLockValidationError); - } - writeContinue(request, request._response); - return createAndStoreObject(bucketName, - bucket, objectKey, objMD, authInfo, canonicalID, cipherBundle, - request, false, streamingV4Params, overheadField, log, 's3:ObjectCreated:Put', next); - }, - ], (err, storingResult) => { - if (err) { - monitoring.promMetrics('PUT', bucketName, err.code, - 'putObject'); - return callback(err, responseHeaders); - } - // ingestSize assumes that these custom headers indicate - // an ingestion PUT which is a metadata only operation. - // Since these headers can be modified client side, they - // should be used with caution if needed for precise - // metrics. - const ingestSize = (request.headers['x-amz-meta-mdonly'] - && !Number.isNaN(request.headers['x-amz-meta-size'])) - ? Number.parseInt(request.headers['x-amz-meta-size'], 10) : null; - const newByteLength = parsedContentLength; + }, + function createCipherBundle(serverSideEncryptionConfig, next) { + if (serverSideEncryptionConfig) { + return kms.createCipherBundle(serverSideEncryptionConfig, log, (err, cipherBundle) => { + if (err) { + return next(err); + } + setSSEHeaders( + responseHeaders, + cipherBundle.algorithm, + cipherBundle.configuredMasterKeyId || cipherBundle.masterKeyId, + ); + return next(null, cipherBundle); + }); + } + return next(null, null); + }, + function objectCreateAndStore(cipherBundle, next) { + const objectLockValidationError = validateHeaders(bucket, headers, log); + if (objectLockValidationError) { + return next(objectLockValidationError); + } + writeContinue(request, request._response); + return createAndStoreObject( + bucketName, + bucket, + objectKey, + objMD, + authInfo, + canonicalID, + cipherBundle, + request, + false, + streamingV4Params, + overheadField, + log, + 's3:ObjectCreated:Put', + next, + ); + }, + ], + (err, storingResult) => { + if (err) { + monitoring.promMetrics('PUT', bucketName, err.code, 'putObject'); + return callback(err, responseHeaders); + } + // ingestSize assumes that these custom headers indicate + // an ingestion PUT which is a metadata only operation. + // Since these headers can be modified client side, they + // should be used with caution if needed for precise + // metrics. + const ingestSize = + request.headers['x-amz-meta-mdonly'] && !Number.isNaN(request.headers['x-amz-meta-size']) + ? Number.parseInt(request.headers['x-amz-meta-size'], 10) + : null; + const newByteLength = parsedContentLength; - setExpirationHeaders(responseHeaders, { - lifecycleConfig: bucket.getLifecycleConfiguration(), - objectParams: { - key: objectKey, - date: storingResult.lastModified, - tags: storingResult.tags, - }, - }); + setExpirationHeaders(responseHeaders, { + lifecycleConfig: bucket.getLifecycleConfiguration(), + objectParams: { + key: objectKey, + date: storingResult.lastModified, + tags: storingResult.tags, + }, + }); - // Utapi expects null or a number for oldByteLength: - // * null - new object - // * 0 or > 0 - existing object with content-length 0 or > 0 - // objMD here is the master version that we would - // have overwritten if there was an existing version or object - // - // TODO: Handle utapi metrics for null version overwrites. - const oldByteLength = objMD && objMD['content-length'] - !== undefined ? objMD['content-length'] : null; - if (storingResult) { - // ETag's hex should always be enclosed in quotes - responseHeaders.ETag = `"${storingResult.contentMD5}"`; - } - const vcfg = bucket.getVersioningConfiguration(); - const isVersionedObj = vcfg && vcfg.Status === 'Enabled'; - if (isVersionedObj) { - if (storingResult && storingResult.versionId) { - responseHeaders['x-amz-version-id'] = - versionIdUtils.encode(storingResult.versionId); - } - } + // Utapi expects null or a number for oldByteLength: + // * null - new object + // * 0 or > 0 - existing object with content-length 0 or > 0 + // objMD here is the master version that we would + // have overwritten if there was an existing version or object + // + // TODO: Handle utapi metrics for null version overwrites. + const oldByteLength = + objMD && objMD['content-length'] !== undefined ? objMD['content-length'] : null; + if (storingResult) { + // ETag's hex should always be enclosed in quotes + responseHeaders.ETag = `"${storingResult.contentMD5}"`; + } + const vcfg = bucket.getVersioningConfiguration(); + const isVersionedObj = vcfg && vcfg.Status === 'Enabled'; + if (isVersionedObj) { + if (storingResult && storingResult.versionId) { + responseHeaders['x-amz-version-id'] = versionIdUtils.encode(storingResult.versionId); + } + } - // Only pre-existing non-versioned objects get 0 all others use 1 - const numberOfObjects = !isVersionedObj && oldByteLength !== null ? 0 : 1; + // Only pre-existing non-versioned objects get 0 all others use 1 + const numberOfObjects = !isVersionedObj && oldByteLength !== null ? 0 : 1; - // only the bucket owner's metrics should be updated, regardless of - // who the requester is - pushMetric('putObject', log, { - authInfo, - canonicalID: bucket.getOwner(), - bucket: bucketName, - keys: [objectKey], - newByteLength, - oldByteLength: isVersionedObj ? null : oldByteLength, - versionId: isVersionedObj && storingResult ? storingResult.versionId : undefined, - location: bucket.getLocationConstraint(), - numberOfObjects, - }); - monitoring.promMetrics('PUT', bucketName, '200', - 'putObject', newByteLength, oldByteLength, isVersionedObj, - null, ingestSize); + // only the bucket owner's metrics should be updated, regardless of + // who the requester is + pushMetric('putObject', log, { + authInfo, + canonicalID: bucket.getOwner(), + bucket: bucketName, + keys: [objectKey], + newByteLength, + oldByteLength: isVersionedObj ? null : oldByteLength, + versionId: isVersionedObj && storingResult ? storingResult.versionId : undefined, + location: bucket.getLocationConstraint(), + numberOfObjects, + }); + monitoring.promMetrics( + 'PUT', + bucketName, + '200', + 'putObject', + newByteLength, + oldByteLength, + isVersionedObj, + null, + ingestSize, + ); - if (isPutVersion) { - const durationMs = Date.now() - new Date(objMD.archive.restoreRequestedAt); - monitoring.lifecycleDuration.observe( - { type: 'restore', location: objMD.dataStoreName }, - durationMs / 1000); - } + if (isPutVersion) { + const durationMs = Date.now() - new Date(objMD.archive.restoreRequestedAt); + monitoring.lifecycleDuration.observe( + { type: 'restore', location: objMD.dataStoreName }, + durationMs / 1000, + ); + } - return callback(null, responseHeaders); - }); - })); + return callback(null, responseHeaders); + }, + ); + }), + ); } module.exports = objectPut; diff --git a/lib/api/objectPutACL.js b/lib/api/objectPutACL.js index 62045f7cc8..a53c8314ce 100644 --- a/lib/api/objectPutACL.js +++ b/lib/api/objectPutACL.js @@ -7,8 +7,11 @@ const { pushMetric } = require('../utapi/utilities'); const collectCorsHeaders = require('../utilities/collectCorsHeaders'); const constants = require('../../constants'); const vault = require('../auth/vault'); -const { decodeVersionId, getVersionIdResHeader, getVersionSpecificMetadataOptions } - = require('./apiUtils/object/versioning'); +const { + decodeVersionId, + getVersionIdResHeader, + getVersionSpecificMetadataOptions, +} = require('./apiUtils/object/versioning'); const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUtils'); const monitoring = require('../utilities/monitoringHandler'); const { config } = require('../Config'); @@ -65,11 +68,7 @@ function objectPutACL(authInfo, request, log, cb) { monitoring.promMetrics('PUT', bucketName, 400, 'putObjectAcl'); return cb(errors.InvalidArgument); } - const possibleGroups = [ - constants.publicId, - constants.allAuthedUsersId, - constants.logId, - ]; + const possibleGroups = [constants.publicId, constants.allAuthedUsersId, constants.logId]; const decodedVidResult = decodeVersionId(request.query); if (decodedVidResult instanceof Error) { @@ -100,224 +99,217 @@ function objectPutACL(authInfo, request, log, cb) { READ_ACP: [], }; - const grantReadHeader = - aclUtils.parseGrant(request.headers['x-amz-grant-read'], 'READ'); - const grantReadACPHeader = - aclUtils.parseGrant(request.headers['x-amz-grant-read-acp'], - 'READ_ACP'); - const grantWriteACPHeader = aclUtils.parseGrant( - request.headers['x-amz-grant-write-acp'], 'WRITE_ACP'); - const grantFullControlHeader = aclUtils.parseGrant( - request.headers['x-amz-grant-full-control'], 'FULL_CONTROL'); + const grantReadHeader = aclUtils.parseGrant(request.headers['x-amz-grant-read'], 'READ'); + const grantReadACPHeader = aclUtils.parseGrant(request.headers['x-amz-grant-read-acp'], 'READ_ACP'); + const grantWriteACPHeader = aclUtils.parseGrant(request.headers['x-amz-grant-write-acp'], 'WRITE_ACP'); + const grantFullControlHeader = aclUtils.parseGrant(request.headers['x-amz-grant-full-control'], 'FULL_CONTROL'); - return async.waterfall([ - function validateBucketAndObj(next) { - return standardMetadataValidateBucketAndObj(metadataValParams, request.actionImplicitDenies, log, - (err, bucket, objectMD) => { - if (err) { - return next(err); - } - if (!objectMD) { - const err = reqVersionId ? errors.NoSuchVersion : - errors.NoSuchKey; - return next(err, bucket); - } - if (objectMD.isDeleteMarker) { - log.trace('delete marker detected', - { method: 'objectPutACL' }); - // FIXME we should return a `x-amz-delete-marker: true` header, - // see S3C-7592 - return next(errors.MethodNotAllowed, bucket); + return async.waterfall( + [ + function validateBucketAndObj(next) { + return standardMetadataValidateBucketAndObj( + metadataValParams, + request.actionImplicitDenies, + log, + (err, bucket, objectMD) => { + if (err) { + return next(err); + } + if (!objectMD) { + const err = reqVersionId ? errors.NoSuchVersion : errors.NoSuchKey; + return next(err, bucket); + } + if (objectMD.isDeleteMarker) { + log.trace('delete marker detected', { method: 'objectPutACL' }); + // FIXME we should return a `x-amz-delete-marker: true` header, + // see S3C-7592 + return next(errors.MethodNotAllowed, bucket); + } + return next(null, bucket, objectMD); + }, + ); + }, + function parseAclFromXml(bucket, objectMD, next) { + // If not setting acl through headers, parse body + let jsonGrants; + let aclOwnerID; + if ( + newCannedACL === undefined && + grantReadHeader === undefined && + grantReadACPHeader === undefined && + grantWriteACPHeader === undefined && + grantFullControlHeader === undefined + ) { + if (request.post) { + log.trace('using acls from request body'); + return aclUtils.parseAclXml(request.post, log, (err, jsonGrants, aclOwnerID) => + next(err, bucket, objectMD, jsonGrants, aclOwnerID), + ); } - return next(null, bucket, objectMD); - }); - }, - function parseAclFromXml(bucket, objectMD, next) { - // If not setting acl through headers, parse body - let jsonGrants; - let aclOwnerID; - if (newCannedACL === undefined - && grantReadHeader === undefined - && grantReadACPHeader === undefined - && grantWriteACPHeader === undefined - && grantFullControlHeader === undefined) { - if (request.post) { - log.trace('using acls from request body'); - return aclUtils.parseAclXml(request.post, log, - (err, jsonGrants, aclOwnerID) => next(err, bucket, - objectMD, jsonGrants, aclOwnerID)); + // If no ACLs sent with request at all + return next(errors.MalformedXML, bucket); } - // If no ACLs sent with request at all - return next(errors.MalformedXML, bucket); - } - /** - * If acl set in headers (including canned acl) pass bucket and - * undefined to the next function - */ - log.debug('using acls from request headers'); - return next(null, bucket, objectMD, jsonGrants, aclOwnerID); - }, - function processAcls(bucket, objectMD, jsonGrants, aclOwnerID, next) { - if (newCannedACL) { - log.debug('canned acl', { cannedAcl: newCannedACL }); - addACLParams.Canned = newCannedACL; - return next(null, bucket, objectMD, addACLParams); - } - let usersIdentifiedByEmail = []; - let usersIdentifiedByGroup = []; - let usersIdentifiedByID = []; - let hasError = false; + /** + * If acl set in headers (including canned acl) pass bucket and + * undefined to the next function + */ + log.debug('using acls from request headers'); + return next(null, bucket, objectMD, jsonGrants, aclOwnerID); + }, + function processAcls(bucket, objectMD, jsonGrants, aclOwnerID, next) { + if (newCannedACL) { + log.debug('canned acl', { cannedAcl: newCannedACL }); + addACLParams.Canned = newCannedACL; + return next(null, bucket, objectMD, addACLParams); + } + let usersIdentifiedByEmail = []; + let usersIdentifiedByGroup = []; + let usersIdentifiedByID = []; + let hasError = false; - // If grants set by xml and xml owner ID is incorrect - if (aclOwnerID && (aclOwnerID !== objectMD['owner-id'])) { - log.trace('incorrect owner ID provided in ACL', { - ACL: request.post, - method: 'objectPutACL', - }); - return next(errors.AccessDenied, bucket); - } + // If grants set by xml and xml owner ID is incorrect + if (aclOwnerID && aclOwnerID !== objectMD['owner-id']) { + log.trace('incorrect owner ID provided in ACL', { + ACL: request.post, + method: 'objectPutACL', + }); + return next(errors.AccessDenied, bucket); + } - /** - * If grants set by xml, loop through the grants - * and separate grant types so parsed in same manner - * as header grants - */ - if (jsonGrants) { - log.trace('parsing acl grants'); - jsonGrants.forEach(grant => { - const grantee = grant.Grantee[0]; - const granteeType = grantee.$['xsi:type']; - const permission = grant.Permission[0]; - let skip = false; - if (possibleGrants.indexOf(permission) < 0) { - skip = true; - } - if (!skip && granteeType === 'AmazonCustomerByEmail') { - usersIdentifiedByEmail.push({ - identifier: grantee.EmailAddress[0], - grantType: permission, - userIDType: 'emailaddress', - }); - } - if (!skip && granteeType === 'CanonicalUser') { - usersIdentifiedByID.push({ - identifier: grantee.ID[0], - grantType: permission, - userIDType: 'id', - }); - } - if (!skip && granteeType === 'Group') { - if (possibleGroups.indexOf(grantee.URI[0]) < 0) { - log.trace('invalid user group', - { userGroup: grantee.URI[0] }); - hasError = true; - return next(errors.InvalidArgument, bucket); + /** + * If grants set by xml, loop through the grants + * and separate grant types so parsed in same manner + * as header grants + */ + if (jsonGrants) { + log.trace('parsing acl grants'); + jsonGrants.forEach(grant => { + const grantee = grant.Grantee[0]; + const granteeType = grantee.$['xsi:type']; + const permission = grant.Permission[0]; + let skip = false; + if (possibleGrants.indexOf(permission) < 0) { + skip = true; + } + if (!skip && granteeType === 'AmazonCustomerByEmail') { + usersIdentifiedByEmail.push({ + identifier: grantee.EmailAddress[0], + grantType: permission, + userIDType: 'emailaddress', + }); + } + if (!skip && granteeType === 'CanonicalUser') { + usersIdentifiedByID.push({ + identifier: grantee.ID[0], + grantType: permission, + userIDType: 'id', + }); + } + if (!skip && granteeType === 'Group') { + if (possibleGroups.indexOf(grantee.URI[0]) < 0) { + log.trace('invalid user group', { userGroup: grantee.URI[0] }); + hasError = true; + return next(errors.InvalidArgument, bucket); + } + return usersIdentifiedByGroup.push({ + identifier: grantee.URI[0], + grantType: permission, + userIDType: 'uri', + }); } - return usersIdentifiedByGroup.push({ - identifier: grantee.URI[0], - grantType: permission, - userIDType: 'uri', - }); + return undefined; + }); + if (hasError) { + return undefined; } - return undefined; - }); - if (hasError) { - return undefined; - } - } else { - // If no canned ACL and no parsed xml, loop - // through the access headers - const allGrantHeaders = - [].concat(grantReadHeader, - grantReadACPHeader, grantWriteACPHeader, - grantFullControlHeader); + } else { + // If no canned ACL and no parsed xml, loop + // through the access headers + const allGrantHeaders = [].concat( + grantReadHeader, + grantReadACPHeader, + grantWriteACPHeader, + grantFullControlHeader, + ); - usersIdentifiedByEmail = allGrantHeaders.filter(item => - item && item.userIDType.toLowerCase() === 'emailaddress'); - usersIdentifiedByGroup = allGrantHeaders - .filter(itm => itm && itm.userIDType - .toLowerCase() === 'uri'); - for (let i = 0; i < usersIdentifiedByGroup.length; i++) { - if (possibleGroups.indexOf( - usersIdentifiedByGroup[i].identifier) < 0) { - log.trace('invalid user group', - { userGroup: usersIdentifiedByGroup[i] - .identifier }); - return next(errors.InvalidArgument, bucket); + usersIdentifiedByEmail = allGrantHeaders.filter( + item => item && item.userIDType.toLowerCase() === 'emailaddress', + ); + usersIdentifiedByGroup = allGrantHeaders.filter( + itm => itm && itm.userIDType.toLowerCase() === 'uri', + ); + for (let i = 0; i < usersIdentifiedByGroup.length; i++) { + if (possibleGroups.indexOf(usersIdentifiedByGroup[i].identifier) < 0) { + log.trace('invalid user group', { userGroup: usersIdentifiedByGroup[i].identifier }); + return next(errors.InvalidArgument, bucket); + } } + /** TODO: Consider whether want to verify with Vault + * whether canonicalID is associated with existing + * account before adding to ACL */ + usersIdentifiedByID = allGrantHeaders.filter( + item => item && item.userIDType.toLowerCase() === 'id', + ); } - /** TODO: Consider whether want to verify with Vault - * whether canonicalID is associated with existing - * account before adding to ACL */ - usersIdentifiedByID = allGrantHeaders - .filter(item => item && item.userIDType - .toLowerCase() === 'id'); - } - const justEmails = usersIdentifiedByEmail - .map(item => item.identifier); - // If have to lookup canonicalID's do that asynchronously - if (justEmails.length > 0) { - return vault.getCanonicalIds( - justEmails, log, (err, results) => { + const justEmails = usersIdentifiedByEmail.map(item => item.identifier); + // If have to lookup canonicalID's do that asynchronously + if (justEmails.length > 0) { + return vault.getCanonicalIds(justEmails, log, (err, results) => { if (err) { - log.trace('error looking up canonical ids', - { error: err, method: 'getCanonicalIDs' }); + log.trace('error looking up canonical ids', { error: err, method: 'getCanonicalIDs' }); return next(err, bucket); } - const reconstructedUsersIdentifiedByEmail = aclUtils - .reconstructUsersIdentifiedByEmail(results, - usersIdentifiedByEmail); + const reconstructedUsersIdentifiedByEmail = aclUtils.reconstructUsersIdentifiedByEmail( + results, + usersIdentifiedByEmail, + ); const allUsers = [].concat( reconstructedUsersIdentifiedByEmail, usersIdentifiedByID, - usersIdentifiedByGroup); - const revisedAddACLParams = aclUtils - .sortHeaderGrants(allUsers, addACLParams); - return next(null, bucket, objectMD, - revisedAddACLParams); + usersIdentifiedByGroup, + ); + const revisedAddACLParams = aclUtils.sortHeaderGrants(allUsers, addACLParams); + return next(null, bucket, objectMD, revisedAddACLParams); }); + } + const allUsers = [].concat(usersIdentifiedByID, usersIdentifiedByGroup); + const revisedAddACLParams = aclUtils.sortHeaderGrants(allUsers, addACLParams); + return next(null, bucket, objectMD, revisedAddACLParams); + }, + function addAclsToObjMD(bucket, objectMD, ACLParams, next) { + // Add acl's to object metadata + const params = getVersionSpecificMetadataOptions(objectMD, config.nullVersionCompatMode); + acl.addObjectACL(bucket, objectKey, objectMD, ACLParams, params, log, err => + next(err, bucket, objectMD), + ); + }, + ], + (err, bucket, objectMD) => { + const resHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + if (err) { + log.trace('error processing request', { + error: err, + method: 'objectPutACL', + }); + monitoring.promMetrics('PUT', bucketName, err.code, 'putObjectAcl'); + return cb(err, resHeaders); } - const allUsers = [].concat( - usersIdentifiedByID, - usersIdentifiedByGroup); - const revisedAddACLParams = - aclUtils.sortHeaderGrants(allUsers, addACLParams); - return next(null, bucket, objectMD, revisedAddACLParams); - }, - function addAclsToObjMD(bucket, objectMD, ACLParams, next) { - // Add acl's to object metadata - const params = getVersionSpecificMetadataOptions(objectMD, config.nullVersionCompatMode); - acl.addObjectACL(bucket, objectKey, objectMD, - ACLParams, params, log, err => next(err, bucket, objectMD)); - }, - ], (err, bucket, objectMD) => { - const resHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); - if (err) { - log.trace('error processing request', { - error: err, - method: 'objectPutACL', - }); - monitoring.promMetrics( - 'PUT', bucketName, err.code, 'putObjectAcl'); - return cb(err, resHeaders); - } - const verCfg = bucket.getVersioningConfiguration(); - resHeaders['x-amz-version-id'] = - getVersionIdResHeader(verCfg, objectMD); + const verCfg = bucket.getVersioningConfiguration(); + resHeaders['x-amz-version-id'] = getVersionIdResHeader(verCfg, objectMD); - log.trace('processed request successfully in object put acl api'); - pushMetric('putObjectAcl', log, { - authInfo, - bucket: bucketName, - keys: [objectKey], - versionId: objectMD ? objectMD.versionId : undefined, - location: objectMD ? objectMD.dataStoreName : undefined, - }); - monitoring.promMetrics('PUT', bucketName, '200', 'putObjectAcl'); - return cb(null, resHeaders); - }); + log.trace('processed request successfully in object put acl api'); + pushMetric('putObjectAcl', log, { + authInfo, + bucket: bucketName, + keys: [objectKey], + versionId: objectMD ? objectMD.versionId : undefined, + location: objectMD ? objectMD.dataStoreName : undefined, + }); + monitoring.promMetrics('PUT', bucketName, '200', 'putObjectAcl'); + return cb(null, resHeaders); + }, + ); } module.exports = objectPutACL; diff --git a/lib/api/objectPutCopyPart.js b/lib/api/objectPutCopyPart.js index 3f3999b212..235343c148 100644 --- a/lib/api/objectPutCopyPart.js +++ b/lib/api/objectPutCopyPart.js @@ -5,8 +5,7 @@ const validateHeaders = s3middleware.validateConditionalHeaders; const collectCorsHeaders = require('../utilities/collectCorsHeaders'); const constants = require('../../constants'); const { data } = require('../data/wrapper'); -const locationConstraintCheck = - require('./apiUtils/object/locationConstraintCheck'); +const locationConstraintCheck = require('./apiUtils/object/locationConstraintCheck'); const metadata = require('../metadata/wrapper'); const { pushMetric } = require('../utapi/utilities'); const services = require('../services'); @@ -35,8 +34,7 @@ const skipError = new Error('skip'); * @param {function} callback - final callback to call with the result * @return {undefined} */ -function objectPutCopyPart(authInfo, request, sourceBucket, - sourceObject, reqVersionId, log, callback) { +function objectPutCopyPart(authInfo, request, sourceBucket, sourceObject, reqVersionId, log, callback) { log.debug('processing request', { method: 'objectPutCopyPart' }); const destBucketName = request.bucketName; const destObjectKey = request.objectKey; @@ -62,8 +60,7 @@ function objectPutCopyPart(authInfo, request, sourceBucket, const partNumber = Number.parseInt(request.query.partNumber, 10); // AWS caps partNumbers at 10,000 if (partNumber > 10000 || !Number.isInteger(partNumber) || partNumber < 1) { - monitoring.promMetrics('PUT', destBucketName, 400, - 'putObjectCopyPart'); + monitoring.promMetrics('PUT', destBucketName, 400, 'putObjectCopyPart'); return callback(errors.InvalidArgument); } // We pad the partNumbers so that the parts will be sorted @@ -72,7 +69,9 @@ function objectPutCopyPart(authInfo, request, sourceBucket, // Note that keys in the query object retain their case, so // request.query.uploadId must be called with that exact // capitalization - const { query: { uploadId } } = request; + const { + query: { uploadId }, + } = request; const valPutParams = { authInfo, @@ -87,10 +86,13 @@ function objectPutCopyPart(authInfo, request, sourceBucket, // as validating for the destination bucket except additionally need // the uploadId and splitter. // Also, requestType is 'putPart or complete' - const valMPUParams = Object.assign({ - uploadId, - splitter: constants.splitter, - }, valPutParams); + const valMPUParams = Object.assign( + { + uploadId, + splitter: constants.splitter, + }, + valPutParams, + ); valMPUParams.requestType = 'putPart or complete'; const dataStoreContext = { @@ -103,131 +105,161 @@ function objectPutCopyPart(authInfo, request, sourceBucket, enableQuota: true, }; - return async.waterfall([ - function checkDestAuth(next) { - return standardMetadataValidateBucketAndObj(valPutParams, request.actionImplicitDenies, log, - (err, destBucketMD) => { - if (err) { - log.debug('error validating authorization for ' + - 'destination bucket', - { error: err }); - return next(err, destBucketMD); - } - const flag = destBucketMD.hasDeletedFlag() - || destBucketMD.hasTransientFlag(); - if (flag) { - log.trace('deleted flag or transient flag ' + - 'on destination bucket', { flag }); - return next(errors.NoSuchBucket); - } - return next(null, destBucketMD); - }); - }, - function checkSourceAuthorization(destBucketMD, next) { - return standardMetadataValidateBucketAndObj({ - ...valGetParams, - serverAccessLogOptions: { copySource: true }, - }, request.actionImplicitDenies, log, - (err, sourceBucketMD, sourceObjMD) => { - if (err) { - log.debug('error validating get part of request', - { error: err }); - // eslint-disable-next-line no-param-reassign - request.sourceServerAccessLog && (request.sourceServerAccessLog.error = err); - return next(err, destBucketMD); - } - if (!sourceObjMD) { - log.debug('no source object', { sourceObject }); - const err = reqVersionId ? errors.NoSuchVersion : - errors.NoSuchKey; - // eslint-disable-next-line no-param-reassign - request.sourceServerAccessLog && (request.sourceServerAccessLog.error = err); - return next(err, destBucketMD); - } - let sourceLocationConstraintName = - sourceObjMD.dataStoreName; - // for backwards compatibility before storing dataStoreName - // TODO: handle in objectMD class - if (!sourceLocationConstraintName && - sourceObjMD.location[0] && - sourceObjMD.location[0].dataStoreName) { - sourceLocationConstraintName = - sourceObjMD.location[0].dataStoreName; - } - // check if object data is in a cold storage - const coldErr = verifyColdObjectAvailable(sourceObjMD); - if (coldErr) { - // eslint-disable-next-line no-param-reassign - request.sourceServerAccessLog && (request.sourceServerAccessLog.error = coldErr); - return next(coldErr, null); - } - if (sourceObjMD.isDeleteMarker) { - log.debug('delete marker on source object', - { sourceObject }); - let err; - if (reqVersionId) { - err = errorInstances.InvalidRequest - .customizeDescription('The source of a copy ' + - 'request may not specifically refer to a delete' + - 'marker by version id.'); - } else { - // if user specifies a key in a versioned source bucket - // without specifying a version, and the object has a - // delete marker, return NoSuchKey - err = errors.NoSuchKey; + return async.waterfall( + [ + function checkDestAuth(next) { + return standardMetadataValidateBucketAndObj( + valPutParams, + request.actionImplicitDenies, + log, + (err, destBucketMD) => { + if (err) { + log.debug('error validating authorization for ' + 'destination bucket', { error: err }); + return next(err, destBucketMD); } - // eslint-disable-next-line no-param-reassign - request.sourceServerAccessLog && (request.sourceServerAccessLog.error = err); - return next(err, destBucketMD); - } - const headerValResult = - validateHeaders(request.headers, - sourceObjMD['last-modified'], - sourceObjMD['content-md5']); - if (headerValResult.error) { - request.sourceServerAccessLog + const flag = destBucketMD.hasDeletedFlag() || destBucketMD.hasTransientFlag(); + if (flag) { + log.trace('deleted flag or transient flag ' + 'on destination bucket', { flag }); + return next(errors.NoSuchBucket); + } + return next(null, destBucketMD); + }, + ); + }, + function checkSourceAuthorization(destBucketMD, next) { + return standardMetadataValidateBucketAndObj( + { + ...valGetParams, + serverAccessLogOptions: { copySource: true }, + }, + request.actionImplicitDenies, + log, + (err, sourceBucketMD, sourceObjMD) => { + if (err) { + log.debug('error validating get part of request', { error: err }); // eslint-disable-next-line no-param-reassign - && (request.sourceServerAccessLog.error = errors.PreconditionFailed); - return next(errors.PreconditionFailed, destBucketMD); - } - const copyLocator = setUpCopyLocator(sourceObjMD, - request.headers['x-amz-copy-source-range'], log); - if (copyLocator.error) { - // eslint-disable-next-line no-param-reassign - request.sourceServerAccessLog && (request.sourceServerAccessLog.error = copyLocator.error); - return next(copyLocator.error, destBucketMD); - } - let sourceVerId; - // If specific version requested, include copy source - // version id in response. Include in request by default - // if versioning is enabled or suspended. - if (sourceBucketMD.getVersioningConfiguration() || - reqVersionId) { - if (sourceObjMD.isNull || !sourceObjMD.versionId) { - sourceVerId = 'null'; - } else { - sourceVerId = - versionIdUtils.encode(sourceObjMD.versionId); + request.sourceServerAccessLog && (request.sourceServerAccessLog.error = err); + return next(err, destBucketMD); } - } - return next(null, copyLocator.dataLocator, destBucketMD, - copyLocator.copyObjectSize, sourceVerId, - sourceLocationConstraintName, sourceObjMD); - }); - }, - function _validateQuotas(dataLocator, destBucketMD, - copyObjectSize, sourceVerId, - sourceLocationConstraintName, sourceObjMD, next) { - return validateQuotas(request, destBucketMD, request.accountQuotas, valPutParams.requestType, - request.apiMethod, sourceObjMD?.['content-length'] || 0, false, log, err => - next(err, dataLocator, destBucketMD, copyObjectSize, sourceVerId, sourceLocationConstraintName)); - }, - // get MPU shadow bucket to get splitter based on MD version - function getMpuShadowBucket(dataLocator, destBucketMD, - copyObjectSize, sourceVerId, - sourceLocationConstraintName, next) { - return metadata.getBucket(mpuBucketName, log, - (err, mpuBucket) => { + if (!sourceObjMD) { + log.debug('no source object', { sourceObject }); + const err = reqVersionId ? errors.NoSuchVersion : errors.NoSuchKey; + // eslint-disable-next-line no-param-reassign + request.sourceServerAccessLog && (request.sourceServerAccessLog.error = err); + return next(err, destBucketMD); + } + let sourceLocationConstraintName = sourceObjMD.dataStoreName; + // for backwards compatibility before storing dataStoreName + // TODO: handle in objectMD class + if ( + !sourceLocationConstraintName && + sourceObjMD.location[0] && + sourceObjMD.location[0].dataStoreName + ) { + sourceLocationConstraintName = sourceObjMD.location[0].dataStoreName; + } + // check if object data is in a cold storage + const coldErr = verifyColdObjectAvailable(sourceObjMD); + if (coldErr) { + // eslint-disable-next-line no-param-reassign + request.sourceServerAccessLog && (request.sourceServerAccessLog.error = coldErr); + return next(coldErr, null); + } + if (sourceObjMD.isDeleteMarker) { + log.debug('delete marker on source object', { sourceObject }); + let err; + if (reqVersionId) { + err = errorInstances.InvalidRequest.customizeDescription( + 'The source of a copy ' + + 'request may not specifically refer to a delete' + + 'marker by version id.', + ); + } else { + // if user specifies a key in a versioned source bucket + // without specifying a version, and the object has a + // delete marker, return NoSuchKey + err = errors.NoSuchKey; + } + // eslint-disable-next-line no-param-reassign + request.sourceServerAccessLog && (request.sourceServerAccessLog.error = err); + return next(err, destBucketMD); + } + const headerValResult = validateHeaders( + request.headers, + sourceObjMD['last-modified'], + sourceObjMD['content-md5'], + ); + if (headerValResult.error) { + request.sourceServerAccessLog && + // eslint-disable-next-line no-param-reassign + (request.sourceServerAccessLog.error = errors.PreconditionFailed); + return next(errors.PreconditionFailed, destBucketMD); + } + const copyLocator = setUpCopyLocator( + sourceObjMD, + request.headers['x-amz-copy-source-range'], + log, + ); + if (copyLocator.error) { + // eslint-disable-next-line no-param-reassign + request.sourceServerAccessLog && (request.sourceServerAccessLog.error = copyLocator.error); + return next(copyLocator.error, destBucketMD); + } + let sourceVerId; + // If specific version requested, include copy source + // version id in response. Include in request by default + // if versioning is enabled or suspended. + if (sourceBucketMD.getVersioningConfiguration() || reqVersionId) { + if (sourceObjMD.isNull || !sourceObjMD.versionId) { + sourceVerId = 'null'; + } else { + sourceVerId = versionIdUtils.encode(sourceObjMD.versionId); + } + } + return next( + null, + copyLocator.dataLocator, + destBucketMD, + copyLocator.copyObjectSize, + sourceVerId, + sourceLocationConstraintName, + sourceObjMD, + ); + }, + ); + }, + function _validateQuotas( + dataLocator, + destBucketMD, + copyObjectSize, + sourceVerId, + sourceLocationConstraintName, + sourceObjMD, + next, + ) { + return validateQuotas( + request, + destBucketMD, + request.accountQuotas, + valPutParams.requestType, + request.apiMethod, + sourceObjMD?.['content-length'] || 0, + false, + log, + err => + next(err, dataLocator, destBucketMD, copyObjectSize, sourceVerId, sourceLocationConstraintName), + ); + }, + // get MPU shadow bucket to get splitter based on MD version + function getMpuShadowBucket( + dataLocator, + destBucketMD, + copyObjectSize, + sourceVerId, + sourceLocationConstraintName, + next, + ) { + return metadata.getBucket(mpuBucketName, log, (err, mpuBucket) => { // TODO: move to `.is` once BKTCLT-9 is done and bumped in Cloudserver if (err && err.NoSuchBucket) { return next(errors.NoSuchUpload); @@ -243,105 +275,140 @@ function objectPutCopyPart(authInfo, request, sourceBucket, if (mpuBucket.getMdBucketModelVersion() < 2) { splitter = constants.oldSplitter; } - return next(null, dataLocator, destBucketMD, - copyObjectSize, sourceVerId, splitter, - sourceLocationConstraintName); + return next( + null, + dataLocator, + destBucketMD, + copyObjectSize, + sourceVerId, + splitter, + sourceLocationConstraintName, + ); }); - }, - // Get MPU overview object to check authorization to put a part - // and to get any object location constraint info - function getMpuOverviewObject(dataLocator, destBucketMD, - copyObjectSize, sourceVerId, splitter, - sourceLocationConstraintName, next) { - const mpuOverviewKey = - `overview${splitter}${destObjectKey}${splitter}${uploadId}`; - return metadata.getObjectMD(mpuBucketName, mpuOverviewKey, - null, log, (err, res) => { - if (err) { - // TODO: move to `.is` once BKTCLT-9 is done and bumped in Cloudserver - if (err.NoSuchKey) { - return next(errors.NoSuchUpload); - } - log.error('error getting overview object from ' + - 'mpu bucket', { - error: err, - method: 'objectPutCopyPart::' + - 'metadata.getObjectMD', - }); - return next(err); - } - const initiatorID = res.initiator.ID; - const requesterID = authInfo.isRequesterAnIAMUser() ? - authInfo.getArn() : authInfo.getCanonicalID(); - if (initiatorID !== requesterID) { - return next(errors.AccessDenied); - } - const destObjLocationConstraint = - res.controllingLocationConstraint; - const sseAlgo = res['x-amz-server-side-encryption']; - const sse = sseAlgo ? { - algorithm: sseAlgo, - masterKeyId: res['x-amz-server-side-encryption-aws-kms-key-id'], - } : null; - return next(null, dataLocator, destBucketMD, - destObjLocationConstraint, copyObjectSize, - sourceVerId, sourceLocationConstraintName, sse, splitter); - }); - }, - function goGetData( - dataLocator, - destBucketMD, - destObjLocationConstraint, - copyObjectSize, - sourceVerId, - sourceLocationConstraintName, - sse, - splitter, - next, - ) { - const originalIdentityAuthzResults = request.actionImplicitDenies; - // eslint-disable-next-line no-param-reassign - delete request.actionImplicitDenies; - data.uploadPartCopy( - request, - log, + }, + // Get MPU overview object to check authorization to put a part + // and to get any object location constraint info + function getMpuOverviewObject( + dataLocator, destBucketMD, + copyObjectSize, + sourceVerId, + splitter, sourceLocationConstraintName, - destObjLocationConstraint, - dataLocator, - dataStoreContext, - locationConstraintCheck, - sse, - (error, eTag, lastModified, serverSideEncryption, locations) => { - // eslint-disable-next-line no-param-reassign - request.actionImplicitDenies = originalIdentityAuthzResults; - if (error) { - if (error.message === 'skip') { - return next(skipError, destBucketMD, eTag, - lastModified, sourceVerId, - serverSideEncryption); + next, + ) { + const mpuOverviewKey = `overview${splitter}${destObjectKey}${splitter}${uploadId}`; + return metadata.getObjectMD(mpuBucketName, mpuOverviewKey, null, log, (err, res) => { + if (err) { + // TODO: move to `.is` once BKTCLT-9 is done and bumped in Cloudserver + if (err.NoSuchKey) { + return next(errors.NoSuchUpload); } - // eslint-disable-next-line no-param-reassign - request.sourceServerAccessLog && (request.sourceServerAccessLog.error = error); - return next(error, destBucketMD); + log.error('error getting overview object from ' + 'mpu bucket', { + error: err, + method: 'objectPutCopyPart::' + 'metadata.getObjectMD', + }); + return next(err); } - return next(null, destBucketMD, locations, eTag, - copyObjectSize, sourceVerId, serverSideEncryption, - lastModified, splitter); + const initiatorID = res.initiator.ID; + const requesterID = authInfo.isRequesterAnIAMUser() ? authInfo.getArn() : authInfo.getCanonicalID(); + if (initiatorID !== requesterID) { + return next(errors.AccessDenied); + } + const destObjLocationConstraint = res.controllingLocationConstraint; + const sseAlgo = res['x-amz-server-side-encryption']; + const sse = sseAlgo + ? { + algorithm: sseAlgo, + masterKeyId: res['x-amz-server-side-encryption-aws-kms-key-id'], + } + : null; + return next( + null, + dataLocator, + destBucketMD, + destObjLocationConstraint, + copyObjectSize, + sourceVerId, + sourceLocationConstraintName, + sse, + splitter, + ); }); - }, - function getExistingPartInfo(destBucketMD, locations, totalHash, - copyObjectSize, sourceVerId, serverSideEncryption, lastModified, - splitter, next) { - const partKey = - `${uploadId}${constants.splitter}${paddedPartNumber}`; - metadata.getObjectMD(mpuBucketName, partKey, {}, log, - (err, result) => { + }, + function goGetData( + dataLocator, + destBucketMD, + destObjLocationConstraint, + copyObjectSize, + sourceVerId, + sourceLocationConstraintName, + sse, + splitter, + next, + ) { + const originalIdentityAuthzResults = request.actionImplicitDenies; + // eslint-disable-next-line no-param-reassign + delete request.actionImplicitDenies; + data.uploadPartCopy( + request, + log, + destBucketMD, + sourceLocationConstraintName, + destObjLocationConstraint, + dataLocator, + dataStoreContext, + locationConstraintCheck, + sse, + (error, eTag, lastModified, serverSideEncryption, locations) => { + // eslint-disable-next-line no-param-reassign + request.actionImplicitDenies = originalIdentityAuthzResults; + if (error) { + if (error.message === 'skip') { + return next( + skipError, + destBucketMD, + eTag, + lastModified, + sourceVerId, + serverSideEncryption, + ); + } + // eslint-disable-next-line no-param-reassign + request.sourceServerAccessLog && (request.sourceServerAccessLog.error = error); + return next(error, destBucketMD); + } + return next( + null, + destBucketMD, + locations, + eTag, + copyObjectSize, + sourceVerId, + serverSideEncryption, + lastModified, + splitter, + ); + }, + ); + }, + function getExistingPartInfo( + destBucketMD, + locations, + totalHash, + copyObjectSize, + sourceVerId, + serverSideEncryption, + lastModified, + splitter, + next, + ) { + const partKey = `${uploadId}${constants.splitter}${paddedPartNumber}`; + metadata.getObjectMD(mpuBucketName, partKey, {}, log, (err, result) => { // If there is nothing being overwritten just move on // TODO: move to `.is` once BKTCLT-9 is done and bumped in Cloudserver if (err && !err.NoSuchKey) { - log.debug('error getting current part (if any)', - { error: err }); + log.debug('error getting current part (if any)', { error: err }); return next(err); } let oldLocations; @@ -352,158 +419,242 @@ function objectPutCopyPart(authInfo, request, sourceBucket, // Pull locations to clean up any potential orphans // in data if object put is an overwrite of // already existing object with same key and part number - oldLocations = Array.isArray(oldLocations) ? - oldLocations : [oldLocations]; + oldLocations = Array.isArray(oldLocations) ? oldLocations : [oldLocations]; } - return next(null, destBucketMD, locations, totalHash, - prevObjectSize, copyObjectSize, sourceVerId, - serverSideEncryption, lastModified, oldLocations, splitter); + return next( + null, + destBucketMD, + locations, + totalHash, + prevObjectSize, + copyObjectSize, + sourceVerId, + serverSideEncryption, + lastModified, + oldLocations, + splitter, + ); }); - }, - function storeNewPartMetadata(destBucketMD, locations, totalHash, - prevObjectSize, copyObjectSize, sourceVerId, serverSideEncryption, - lastModified, oldLocations, splitter, next) { - const metaStoreParams = { - partNumber: paddedPartNumber, - contentMD5: totalHash, - size: copyObjectSize, - uploadId, - splitter: constants.splitter, + }, + function storeNewPartMetadata( + destBucketMD, + locations, + totalHash, + prevObjectSize, + copyObjectSize, + sourceVerId, + serverSideEncryption, lastModified, - overheadField: constants.overheadField, - ownerId: destBucketMD.getOwner(), - }; - return services.metadataStorePart(mpuBucketName, - locations, metaStoreParams, log, err => { + oldLocations, + splitter, + next, + ) { + const metaStoreParams = { + partNumber: paddedPartNumber, + contentMD5: totalHash, + size: copyObjectSize, + uploadId, + splitter: constants.splitter, + lastModified, + overheadField: constants.overheadField, + ownerId: destBucketMD.getOwner(), + }; + return services.metadataStorePart(mpuBucketName, locations, metaStoreParams, log, err => { if (err) { - log.debug('error storing new metadata', - { error: err, method: 'storeNewPartMetadata' }); + log.debug('error storing new metadata', { error: err, method: 'storeNewPartMetadata' }); return next(err); } - return next(null, locations, oldLocations, destBucketMD, totalHash, - lastModified, sourceVerId, serverSideEncryption, - prevObjectSize, copyObjectSize, splitter); + return next( + null, + locations, + oldLocations, + destBucketMD, + totalHash, + lastModified, + sourceVerId, + serverSideEncryption, + prevObjectSize, + copyObjectSize, + splitter, + ); }); - }, - function checkCanDeleteOldLocations(partLocations, oldLocations, destBucketMD, - totalHash, lastModified, sourceVerId, serverSideEncryption, - prevObjectSize, copyObjectSize, splitter, next) { - if (!oldLocations) { - return next(null, oldLocations, destBucketMD, totalHash, - lastModified, sourceVerId, serverSideEncryption, - prevObjectSize, copyObjectSize); - } - return services.isCompleteMPUInProgress({ - bucketName: destBucketName, - objectKey: destObjectKey, - uploadId, + }, + function checkCanDeleteOldLocations( + partLocations, + oldLocations, + destBucketMD, + totalHash, + lastModified, + sourceVerId, + serverSideEncryption, + prevObjectSize, + copyObjectSize, splitter, - }, log, (err, completeInProgress) => { - if (err) { - return next(err, destBucketMD); + next, + ) { + if (!oldLocations) { + return next( + null, + oldLocations, + destBucketMD, + totalHash, + lastModified, + sourceVerId, + serverSideEncryption, + prevObjectSize, + copyObjectSize, + ); } - let oldLocationsToDelete = oldLocations; - // Prevent deletion of old data if a completeMPU - // is already in progress because then there is no - // guarantee that the old location will not be the - // committed one. - if (completeInProgress) { - log.warn('not deleting old locations because CompleteMPU is in progress', { - method: 'objectPutCopyPart::checkCanDeleteOldLocations', + return services.isCompleteMPUInProgress( + { bucketName: destBucketName, objectKey: destObjectKey, uploadId, - partLocations, - oldLocations, - }); - oldLocationsToDelete = null; - } - return next(null, oldLocationsToDelete, destBucketMD, totalHash, - lastModified, sourceVerId, serverSideEncryption, - prevObjectSize, copyObjectSize); - }); - }, - function cleanupExistingData(oldLocationsToDelete, destBucketMD, totalHash, - lastModified, sourceVerId, serverSideEncryption, - prevObjectSize, copyObjectSize, next) { - // Clean up the old data now that new metadata (with new - // data locations) has been stored - if (oldLocationsToDelete) { - return data.batchDelete(oldLocationsToDelete, request.method, null, - log, err => { + splitter, + }, + log, + (err, completeInProgress) => { + if (err) { + return next(err, destBucketMD); + } + let oldLocationsToDelete = oldLocations; + // Prevent deletion of old data if a completeMPU + // is already in progress because then there is no + // guarantee that the old location will not be the + // committed one. + if (completeInProgress) { + log.warn('not deleting old locations because CompleteMPU is in progress', { + method: 'objectPutCopyPart::checkCanDeleteOldLocations', + bucketName: destBucketName, + objectKey: destObjectKey, + uploadId, + partLocations, + oldLocations, + }); + oldLocationsToDelete = null; + } + return next( + null, + oldLocationsToDelete, + destBucketMD, + totalHash, + lastModified, + sourceVerId, + serverSideEncryption, + prevObjectSize, + copyObjectSize, + ); + }, + ); + }, + function cleanupExistingData( + oldLocationsToDelete, + destBucketMD, + totalHash, + lastModified, + sourceVerId, + serverSideEncryption, + prevObjectSize, + copyObjectSize, + next, + ) { + // Clean up the old data now that new metadata (with new + // data locations) has been stored + if (oldLocationsToDelete) { + return data.batchDelete(oldLocationsToDelete, request.method, null, log, err => { if (err) { // if error, log the error and move on as it is not // relevant to the client as the client's // object already succeeded putting data, metadata - log.error('error deleting existing data', - { error: err }); + log.error('error deleting existing data', { error: err }); } - return next(null, destBucketMD, totalHash, - lastModified, sourceVerId, serverSideEncryption, - prevObjectSize, copyObjectSize); + return next( + null, + destBucketMD, + totalHash, + lastModified, + sourceVerId, + serverSideEncryption, + prevObjectSize, + copyObjectSize, + ); }); - } - return next(null, destBucketMD, totalHash, - lastModified, sourceVerId, serverSideEncryption, - prevObjectSize, copyObjectSize); - }, - ], (err, destBucketMD, totalHash, lastModified, sourceVerId, - serverSideEncryption, prevObjectSize, copyObjectSize) => { - const corsHeaders = collectCorsHeaders(request.headers.origin, - request.method, destBucketMD); + } + return next( + null, + destBucketMD, + totalHash, + lastModified, + sourceVerId, + serverSideEncryption, + prevObjectSize, + copyObjectSize, + ); + }, + ], + ( + err, + destBucketMD, + totalHash, + lastModified, + sourceVerId, + serverSideEncryption, + prevObjectSize, + copyObjectSize, + ) => { + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, destBucketMD); - // Store full object size for server access logs - if (request.serverAccessLog) { - // eslint-disable-next-line no-param-reassign - request.serverAccessLog.objectSize = copyObjectSize; - } + // Store full object size for server access logs + if (request.serverAccessLog) { + // eslint-disable-next-line no-param-reassign + request.serverAccessLog.objectSize = copyObjectSize; + } - // Initialize the queue for internal log request logging - initializeInternalLogRequestQueue(request); - // Queue the source-side access log (REST.COPY.PART_GET) - queueInternalLogRequest(request, { - operation: 'REST.COPY.PART_GET', - sourceBucket, - sourceObject, - objectSize: copyObjectSize || null, - }); + // Initialize the queue for internal log request logging + initializeInternalLogRequestQueue(request); + // Queue the source-side access log (REST.COPY.PART_GET) + queueInternalLogRequest(request, { + operation: 'REST.COPY.PART_GET', + sourceBucket, + sourceObject, + objectSize: copyObjectSize || null, + }); - if (err && err !== skipError) { - log.trace('error from copy part waterfall', - { error: err }); - monitoring.promMetrics('PUT', destBucketName, err.code, - 'putObjectCopyPart'); - return callback(err, null, corsHeaders); - } - const xml = [ - '', - '', - '', new Date(lastModified) - .toISOString(), '', - '"', totalHash, '"', - '', - ].join(''); + if (err && err !== skipError) { + log.trace('error from copy part waterfall', { error: err }); + monitoring.promMetrics('PUT', destBucketName, err.code, 'putObjectCopyPart'); + return callback(err, null, corsHeaders); + } + const xml = [ + '', + '', + '', + new Date(lastModified).toISOString(), + '', + '"', + totalHash, + '"', + '', + ].join(''); - const additionalHeaders = corsHeaders || {}; - if (serverSideEncryption) { - setSSEHeaders(additionalHeaders, - serverSideEncryption.algorithm, - serverSideEncryption.masterKeyId); - } - additionalHeaders['x-amz-copy-source-version-id'] = sourceVerId; - pushMetric('uploadPartCopy', log, { - authInfo, - canonicalID: destBucketMD.getOwner(), - bucket: destBucketName, - keys: [destObjectKey], - newByteLength: copyObjectSize, - oldByteLength: prevObjectSize, - location: destBucketMD.getLocationConstraint(), - }); - monitoring.promMetrics( - 'PUT', destBucketName, '200', 'putObjectCopyPart'); - return callback(null, xml, additionalHeaders); - }); + const additionalHeaders = corsHeaders || {}; + if (serverSideEncryption) { + setSSEHeaders(additionalHeaders, serverSideEncryption.algorithm, serverSideEncryption.masterKeyId); + } + additionalHeaders['x-amz-copy-source-version-id'] = sourceVerId; + pushMetric('uploadPartCopy', log, { + authInfo, + canonicalID: destBucketMD.getOwner(), + bucket: destBucketName, + keys: [destObjectKey], + newByteLength: copyObjectSize, + oldByteLength: prevObjectSize, + location: destBucketMD.getLocationConstraint(), + }); + monitoring.promMetrics('PUT', destBucketName, '200', 'putObjectCopyPart'); + return callback(null, xml, additionalHeaders); + }, + ); } module.exports = objectPutCopyPart; diff --git a/lib/api/objectPutLegalHold.js b/lib/api/objectPutLegalHold.js index c16f2c84e8..d687f77ce6 100644 --- a/lib/api/objectPutLegalHold.js +++ b/lib/api/objectPutLegalHold.js @@ -2,8 +2,11 @@ const async = require('async'); const { errors, errorInstances, s3middleware } = require('arsenal'); const collectCorsHeaders = require('../utilities/collectCorsHeaders'); -const { decodeVersionId, getVersionIdResHeader, getVersionSpecificMetadataOptions } = - require('./apiUtils/object/versioning'); +const { + decodeVersionId, + getVersionIdResHeader, + getVersionSpecificMetadataOptions, +} = require('./apiUtils/object/versioning'); const getReplicationInfo = require('./apiUtils/object/getReplicationInfo'); const metadata = require('../metadata/wrapper'); const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUtils'); @@ -47,78 +50,87 @@ function objectPutLegalHold(authInfo, request, log, callback) { request, }; - return async.waterfall([ - next => standardMetadataValidateBucketAndObj(metadataValParams, request.actionImplicitDenies, log, + return async.waterfall( + [ + next => + standardMetadataValidateBucketAndObj( + metadataValParams, + request.actionImplicitDenies, + log, + (err, bucket, objectMD) => { + if (err) { + log.trace('request authorization failed', { method: 'objectPutLegalHold', error: err }); + return next(err); + } + if (!objectMD) { + const err = versionId ? errors.NoSuchVersion : errors.NoSuchKey; + log.trace('error no object metadata found', { method: 'objectPutLegalHold', error: err }); + return next(err, bucket); + } + if (objectMD.isDeleteMarker) { + log.trace('version is a delete marker', { method: 'objectPutLegalHold' }); + // FIXME we should return a `x-amz-delete-marker: true` header, + // see S3C-7592 + return next(errors.MethodNotAllowed, bucket); + } + if (!bucket.isObjectLockEnabled()) { + log.trace('object lock not enabled on bucket', { method: 'objectPutLegalHold' }); + return next( + errorInstances.InvalidRequest.customizeDescription( + 'Bucket is missing Object Lock Configuration', + ), + bucket, + ); + } + return next(null, bucket, objectMD); + }, + ), + (bucket, objectMD, next) => { + log.trace('parsing legal hold'); + parseLegalHoldXml(request.post, log, (err, res) => next(err, bucket, res, objectMD)); + }, + (bucket, legalHold, objectMD, next) => { + // eslint-disable-next-line no-param-reassign + objectMD.legalHold = legalHold; + const params = getVersionSpecificMetadataOptions(objectMD, config.nullVersionCompatMode); + const replicationInfo = getReplicationInfo( + config, + objectKey, + bucket, + true, + 0, + REPLICATION_ACTION, + objectMD, + ); + if (replicationInfo) { + // eslint-disable-next-line no-param-reassign + objectMD.replicationInfo = Object.assign({}, objectMD.replicationInfo, replicationInfo); + } + // eslint-disable-next-line no-param-reassign + objectMD.originOp = 's3:ObjectLegalHold:Put'; + metadata.putObjectMD(bucket.getName(), objectKey, objectMD, params, log, err => + next(err, bucket, objectMD), + ); + }, + ], (err, bucket, objectMD) => { + const additionalResHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); if (err) { - log.trace('request authorization failed', - { method: 'objectPutLegalHold', error: err }); - return next(err); - } - if (!objectMD) { - const err = versionId ? errors.NoSuchVersion : - errors.NoSuchKey; - log.trace('error no object metadata found', - { method: 'objectPutLegalHold', error: err }); - return next(err, bucket); - } - if (objectMD.isDeleteMarker) { - log.trace('version is a delete marker', - { method: 'objectPutLegalHold' }); - // FIXME we should return a `x-amz-delete-marker: true` header, - // see S3C-7592 - return next(errors.MethodNotAllowed, bucket); - } - if (!bucket.isObjectLockEnabled()) { - log.trace('object lock not enabled on bucket', - { method: 'objectPutLegalHold' }); - return next(errorInstances.InvalidRequest.customizeDescription( - 'Bucket is missing Object Lock Configuration' - ), bucket); - } - return next(null, bucket, objectMD); - }), - (bucket, objectMD, next) => { - log.trace('parsing legal hold'); - parseLegalHoldXml(request.post, log, (err, res) => - next(err, bucket, res, objectMD)); - }, - (bucket, legalHold, objectMD, next) => { - // eslint-disable-next-line no-param-reassign - objectMD.legalHold = legalHold; - const params = getVersionSpecificMetadataOptions(objectMD, config.nullVersionCompatMode); - const replicationInfo = getReplicationInfo(config, - objectKey, bucket, true, 0, REPLICATION_ACTION, objectMD); - if (replicationInfo) { - // eslint-disable-next-line no-param-reassign - objectMD.replicationInfo = Object.assign({}, - objectMD.replicationInfo, replicationInfo); + log.trace('error processing request', { error: err, method: 'objectPutLegalHold' }); + } else { + pushMetric('putObjectLegalHold', log, { + authInfo, + bucket: bucketName, + keys: [objectKey], + versionId: objectMD ? objectMD.versionId : undefined, + location: objectMD ? objectMD.dataStoreName : undefined, + }); + const verCfg = bucket.getVersioningConfiguration(); + additionalResHeaders['x-amz-version-id'] = getVersionIdResHeader(verCfg, objectMD); } - // eslint-disable-next-line no-param-reassign - objectMD.originOp = 's3:ObjectLegalHold:Put'; - metadata.putObjectMD(bucket.getName(), objectKey, objectMD, params, - log, err => next(err, bucket, objectMD)); + return callback(err, additionalResHeaders); }, - ], (err, bucket, objectMD) => { - const additionalResHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); - if (err) { - log.trace('error processing request', - { error: err, method: 'objectPutLegalHold' }); - } else { - pushMetric('putObjectLegalHold', log, { - authInfo, - bucket: bucketName, - keys: [objectKey], - versionId: objectMD ? objectMD.versionId : undefined, - location: objectMD ? objectMD.dataStoreName : undefined, - }); - const verCfg = bucket.getVersioningConfiguration(); - additionalResHeaders['x-amz-version-id'] = - getVersionIdResHeader(verCfg, objectMD); - } - return callback(err, additionalResHeaders); - }); + ); } module.exports = objectPutLegalHold; diff --git a/lib/api/objectPutPart.js b/lib/api/objectPutPart.js index bf7ba3e367..cf2c5bde80 100644 --- a/lib/api/objectPutPart.js +++ b/lib/api/objectPutPart.js @@ -6,14 +6,12 @@ const collectCorsHeaders = require('../utilities/collectCorsHeaders'); const constants = require('../../constants'); const { data } = require('../data/wrapper'); const { dataStore } = require('./apiUtils/object/storeObject'); -const { isBucketAuthorized } = - require('./apiUtils/authorization/permissionChecks'); +const { isBucketAuthorized } = require('./apiUtils/authorization/permissionChecks'); const kms = require('../kms/wrapper'); const metadata = require('../metadata/wrapper'); const { pushMetric } = require('../utapi/utilities'); const services = require('../services'); -const locationConstraintCheck - = require('./apiUtils/object/locationConstraintCheck'); +const locationConstraintCheck = require('./apiUtils/object/locationConstraintCheck'); const monitoring = require('../utilities/monitoringHandler'); const { config } = require('../Config'); const { BackendInfo } = models; @@ -58,8 +56,7 @@ function _getPartKey(uploadId, splitter, paddedPartNumber) { * @param {function} cb - final callback to call with the result * @return {undefined} */ -function objectPutPart(authInfo, request, streamingV4Params, log, - cb) { +function objectPutPart(authInfo, request, streamingV4Params, log, cb) { log.debug('processing request', { method: 'objectPutPart' }); const size = request.parsedContentLength; @@ -68,8 +65,7 @@ function objectPutPart(authInfo, request, streamingV4Params, log, if (Number.parseInt(size, 10) > constants.maximumAllowedPartSize) { log.debug('put part size too large', { size }); - monitoring.promMetrics('PUT', request.bucketName, 400, - 'putObjectPart'); + monitoring.promMetrics('PUT', request.bucketName, 400, 'putObjectPart'); return cb(errors.EntityTooLarge); } @@ -87,13 +83,11 @@ function objectPutPart(authInfo, request, streamingV4Params, log, const partNumber = Number.parseInt(request.query.partNumber, 10); // AWS caps partNumbers at 10,000 if (partNumber > 10000) { - monitoring.promMetrics('PUT', request.bucketName, 400, - 'putObjectPart'); + monitoring.promMetrics('PUT', request.bucketName, 400, 'putObjectPart'); return cb(errors.TooManyParts); } if (!Number.isInteger(partNumber) || partNumber < 1) { - monitoring.promMetrics('PUT', request.bucketName, 400, - 'putObjectPart'); + monitoring.promMetrics('PUT', request.bucketName, 400, 'putObjectPart'); return cb(errors.InvalidArgument); } const bucketName = request.bucketName; @@ -105,7 +99,9 @@ function objectPutPart(authInfo, request, streamingV4Params, log, }); // Note that keys in the query object retain their case, so // `request.query.uploadId` must be called with that exact capitalization. - const { query: { uploadId } } = request; + const { + query: { uploadId }, + } = request; const mpuBucketName = `${constants.mpuBucketPrefix}${bucketName}`; const { objectKey } = request; const originalIdentityAuthzResults = request.actionImplicitDenies; @@ -113,74 +109,94 @@ function objectPutPart(authInfo, request, streamingV4Params, log, // `requestType` is the general 'objectPut'. const requestType = request.apiMethods || 'objectPutPart'; - return async.waterfall([ - // Get the destination bucket. - next => metadata.getBucket(bucketName, log, - (err, destinationBucket, raftSessionId) => { - if (err?.is?.NoSuchBucket) { - return next(errors.NoSuchBucket, destinationBucket); - } - if (err) { - log.error('error getting the destination bucket', { - error: err, - method: 'objectPutPart::metadata.getBucket', - }); - return next(err, destinationBucket); + return async.waterfall( + [ + // Get the destination bucket. + next => + metadata.getBucket(bucketName, log, (err, destinationBucket, raftSessionId) => { + if (err?.is?.NoSuchBucket) { + return next(errors.NoSuchBucket, destinationBucket); + } + if (err) { + log.error('error getting the destination bucket', { + error: err, + method: 'objectPutPart::metadata.getBucket', + }); + return next(err, destinationBucket); + } + storeServerAccessLogInfo(request, destinationBucket, raftSessionId); + return next(null, destinationBucket); + }), + // Check the bucket authorization. + (destinationBucket, next) => { + if ( + !isBucketAuthorized( + destinationBucket, + requestType, + canonicalID, + authInfo, + log, + request, + request.actionImplicitDenies, + ) + ) { + log.debug('access denied for user on bucket', { requestType }); + return next(errors.AccessDenied, destinationBucket); } - storeServerAccessLogInfo(request, destinationBucket, raftSessionId); return next(null, destinationBucket); - }), - // Check the bucket authorization. - (destinationBucket, next) => { - if (!isBucketAuthorized(destinationBucket, requestType, canonicalID, authInfo, - log, request, request.actionImplicitDenies)) { - log.debug('access denied for user on bucket', { requestType }); - return next(errors.AccessDenied, destinationBucket); - } - return next(null, destinationBucket); - }, - (destinationBucket, next) => validateQuotas(request, destinationBucket, request.accountQuotas, - requestType, request.apiMethod, size, isPutVersion, log, err => next(err, destinationBucket)), - // Validate that no object SSE is provided for part. - // Part must use SSE from initiateMPU (overview in metadata) - (destinationBucket, next) => { - const { error, objectSSE } = parseObjectEncryptionHeaders(request.headers); - if (error) { - return next(error, destinationBucket); - } - if (objectSSE.algorithm) { - return next(errors.InvalidArgument.customizeDescription( - 'x-amz-server-side-encryption header is not supported for this operation.')); - } - return next(null, destinationBucket); - }, - // Get the MPU shadow bucket. - (destinationBucket, next) => - metadata.getBucket(mpuBucketName, log, - (err, mpuBucket) => { - if (err?.is?.NoSuchBucket) { - return next(errors.NoSuchUpload, destinationBucket); - } - if (err) { - log.error('error getting the shadow mpu bucket', { - error: err, - method: 'objectPutPart::metadata.getBucket', - }); - return next(err, destinationBucket); + }, + (destinationBucket, next) => + validateQuotas( + request, + destinationBucket, + request.accountQuotas, + requestType, + request.apiMethod, + size, + isPutVersion, + log, + err => next(err, destinationBucket), + ), + // Validate that no object SSE is provided for part. + // Part must use SSE from initiateMPU (overview in metadata) + (destinationBucket, next) => { + const { error, objectSSE } = parseObjectEncryptionHeaders(request.headers); + if (error) { + return next(error, destinationBucket); } - let splitter = constants.splitter; - // BACKWARD: Remove to remove the old splitter - if (mpuBucket.getMdBucketModelVersion() < 2) { - splitter = constants.oldSplitter; + if (objectSSE.algorithm) { + return next( + errors.InvalidArgument.customizeDescription( + 'x-amz-server-side-encryption header is not supported for this operation.', + ), + ); } - return next(null, destinationBucket, splitter); - }), - // Check authorization of the MPU shadow bucket. - (destinationBucket, splitter, next) => { - const mpuOverviewKey = _getOverviewKey(splitter, objectKey, - uploadId); - return metadata.getObjectMD(mpuBucketName, mpuOverviewKey, {}, log, - (err, res) => { + return next(null, destinationBucket); + }, + // Get the MPU shadow bucket. + (destinationBucket, next) => + metadata.getBucket(mpuBucketName, log, (err, mpuBucket) => { + if (err?.is?.NoSuchBucket) { + return next(errors.NoSuchUpload, destinationBucket); + } + if (err) { + log.error('error getting the shadow mpu bucket', { + error: err, + method: 'objectPutPart::metadata.getBucket', + }); + return next(err, destinationBucket); + } + let splitter = constants.splitter; + // BACKWARD: Remove to remove the old splitter + if (mpuBucket.getMdBucketModelVersion() < 2) { + splitter = constants.oldSplitter; + } + return next(null, destinationBucket, splitter); + }), + // Check authorization of the MPU shadow bucket. + (destinationBucket, splitter, next) => { + const mpuOverviewKey = _getOverviewKey(splitter, objectKey, uploadId); + return metadata.getObjectMD(mpuBucketName, mpuOverviewKey, {}, log, (err, res) => { if (err) { log.error('error getting the object from mpu bucket', { error: err, @@ -189,85 +205,88 @@ function objectPutPart(authInfo, request, streamingV4Params, log, return next(err, destinationBucket); } const initiatorID = res.initiator.ID; - const requesterID = authInfo.isRequesterAnIAMUser() ? - authInfo.getArn() : authInfo.getCanonicalID(); + const requesterID = authInfo.isRequesterAnIAMUser() ? authInfo.getArn() : authInfo.getCanonicalID(); if (initiatorID !== requesterID) { return next(errors.AccessDenied, destinationBucket); } - const objectLocationConstraint = - res.controllingLocationConstraint; + const objectLocationConstraint = res.controllingLocationConstraint; const sseAlgo = res['x-amz-server-side-encryption']; - const sse = sseAlgo ? { - algorithm: sseAlgo, - masterKeyId: res['x-amz-server-side-encryption-aws-kms-key-id'], - } : null; - return next(null, destinationBucket, - objectLocationConstraint, - sse, splitter); + const sse = sseAlgo + ? { + algorithm: sseAlgo, + masterKeyId: res['x-amz-server-side-encryption-aws-kms-key-id'], + } + : null; + return next(null, destinationBucket, objectLocationConstraint, sse, splitter); }); - }, - // Use MPU overview SSE config - (destinationBucket, objectLocationConstraint, encryption, splitter, next) => { - // If MPU has server-side encryption, pass the `res` value - if (encryption) { - return kms.createCipherBundle(encryption, log, (err, res) => { - if (err) { - log.error('error processing the cipher bundle for ' + - 'the destination bucket', { - error: err, - }); - return next(err, destinationBucket); - } - return next(null, destinationBucket, objectLocationConstraint, res, splitter); - // Allow KMS to use a key from previous provider (if sseMigration configured) - // Because ongoing MPU started before sseMigration is no migrated - }, { previousOk: true }); - } - // The MPU does not have server-side encryption, so pass `null` - return next(null, destinationBucket, objectLocationConstraint, null, splitter); - }, - // If data backend is backend that handles mpu (like real AWS), - // no need to store part info in metadata - (destinationBucket, objectLocationConstraint, cipherBundle, - splitter, next) => { - const mpuInfo = { - destinationBucket, - size, - objectKey, - uploadId, - partNumber, - bucketName, - }; - // eslint-disable-next-line no-param-reassign - delete request.actionImplicitDenies; - writeContinue(request, request._response); - return data.putPart(request, mpuInfo, streamingV4Params, - objectLocationConstraint, locationConstraintCheck, log, - (err, partInfo, updatedObjectLC) => { - if (err) { - return next(err, destinationBucket); + }, + // Use MPU overview SSE config + (destinationBucket, objectLocationConstraint, encryption, splitter, next) => { + // If MPU has server-side encryption, pass the `res` value + if (encryption) { + return kms.createCipherBundle( + encryption, + log, + (err, res) => { + if (err) { + log.error('error processing the cipher bundle for ' + 'the destination bucket', { + error: err, + }); + return next(err, destinationBucket); + } + return next(null, destinationBucket, objectLocationConstraint, res, splitter); + // Allow KMS to use a key from previous provider (if sseMigration configured) + // Because ongoing MPU started before sseMigration is no migrated + }, + { previousOk: true }, + ); } - // if data backend handles mpu, skip to end of waterfall - // TODO CLDSRV-640 (artesca) data backend should return SSE to include in response headers - if (partInfo && partInfo.dataStoreType === 'aws_s3') { - return next(skipError, destinationBucket, - partInfo.dataStoreETag); - } - // partInfo will be null if data backend is not external - // if the object location constraint undefined because - // mpu was initiated in legacy version, update it - return next(null, destinationBucket, updatedObjectLC, - cipherBundle, splitter, partInfo); - }); - }, - // Get any pre-existing part. - (destinationBucket, objectLocationConstraint, cipherBundle, - splitter, partInfo, next) => { - const paddedPartNumber = _getPaddedPartNumber(partNumber); - const partKey = _getPartKey(uploadId, splitter, paddedPartNumber); - return metadata.getObjectMD(mpuBucketName, partKey, {}, log, - (err, res) => { + // The MPU does not have server-side encryption, so pass `null` + return next(null, destinationBucket, objectLocationConstraint, null, splitter); + }, + // If data backend is backend that handles mpu (like real AWS), + // no need to store part info in metadata + (destinationBucket, objectLocationConstraint, cipherBundle, splitter, next) => { + const mpuInfo = { + destinationBucket, + size, + objectKey, + uploadId, + partNumber, + bucketName, + }; + // eslint-disable-next-line no-param-reassign + delete request.actionImplicitDenies; + writeContinue(request, request._response); + return data.putPart( + request, + mpuInfo, + streamingV4Params, + objectLocationConstraint, + locationConstraintCheck, + log, + (err, partInfo, updatedObjectLC) => { + if (err) { + return next(err, destinationBucket); + } + // if data backend handles mpu, skip to end of waterfall + // TODO CLDSRV-640 (artesca) data backend should return SSE to include in response headers + if (partInfo && partInfo.dataStoreType === 'aws_s3') { + return next(skipError, destinationBucket, partInfo.dataStoreETag); + } + // partInfo will be null if data backend is not external + // if the object location constraint undefined because + // mpu was initiated in legacy version, update it + return next(null, destinationBucket, updatedObjectLC, cipherBundle, splitter, partInfo); + }, + ); + }, + // Get any pre-existing part. + (destinationBucket, objectLocationConstraint, cipherBundle, splitter, partInfo, next) => { + const paddedPartNumber = _getPaddedPartNumber(partNumber); + const partKey = _getPartKey(uploadId, splitter, paddedPartNumber); + return metadata.getObjectMD(mpuBucketName, partKey, {}, log, (err, res) => { // If there is no object with the same key, continue. if (err && !err.is.NoSuchKey) { log.error('error getting current part (if any)', { @@ -285,78 +304,124 @@ function objectPutPart(authInfo, request, streamingV4Params, log, // Pull locations to clean up any potential orphans in // data if object put is an overwrite of a pre-existing // object with the same key and part number. - oldLocations = Array.isArray(res.partLocations) ? - res.partLocations : [res.partLocations]; - } - return next(null, destinationBucket, - objectLocationConstraint, cipherBundle, - partKey, prevObjectSize, oldLocations, partInfo, splitter); - }); - }, - // Store in data backend. - (destinationBucket, objectLocationConstraint, cipherBundle, - partKey, prevObjectSize, oldLocations, partInfo, splitter, next) => { - // NOTE: set oldLocations to null so we do not batchDelete for now - if (partInfo && - constants.skipBatchDeleteBackends[partInfo.dataStoreType]) { - // skip to storing metadata - return next(null, destinationBucket, partInfo, - partInfo.dataStoreETag, - cipherBundle, partKey, prevObjectSize, null, - objectLocationConstraint, splitter); - } - const objectContext = { - bucketName, - owner: canonicalID, - namespace: request.namespace, - objectKey, - partNumber: _getPaddedPartNumber(partNumber), - uploadId, - }; - const backendInfo = new BackendInfo(config, - objectLocationConstraint); - return dataStore(objectContext, cipherBundle, request, - size, streamingV4Params, backendInfo, log, - (err, dataGetInfo, hexDigest) => { - if (err) { - return next(err, destinationBucket); + oldLocations = Array.isArray(res.partLocations) ? res.partLocations : [res.partLocations]; } - return next(null, destinationBucket, dataGetInfo, hexDigest, - cipherBundle, partKey, prevObjectSize, oldLocations, - objectLocationConstraint, splitter); + return next( + null, + destinationBucket, + objectLocationConstraint, + cipherBundle, + partKey, + prevObjectSize, + oldLocations, + partInfo, + splitter, + ); }); - }, - // Store data locations in metadata and delete any overwritten - // data if completeMPU hasn't been initiated yet. - (destinationBucket, dataGetInfo, hexDigest, cipherBundle, partKey, - prevObjectSize, oldLocations, objectLocationConstraint, splitter, next) => { - // Use an array to be consistent with objectPutCopyPart where there - // could be multiple locations. - const partLocations = [dataGetInfo]; - const sseHeaders = {}; - if (cipherBundle) { - const { algorithm, masterKeyId, cryptoScheme, - cipheredDataKey } = cipherBundle; - partLocations[0].sseAlgorithm = algorithm; - partLocations[0].sseMasterKeyId = masterKeyId; - partLocations[0].sseCryptoScheme = cryptoScheme; - partLocations[0].sseCipheredDataKey = cipheredDataKey; - sseHeaders.algo = algorithm; - sseHeaders.kmsKey = masterKeyId; - } - const omVal = { - // back to Version 3 since number-subparts is not needed - 'md-model-version': 3, - partLocations, - 'key': partKey, - 'last-modified': new Date().toJSON(), - 'content-md5': hexDigest, - 'content-length': size, - 'owner-id': destinationBucket.getOwner(), - }; - const mdParams = { overheadField: constants.overheadField }; - return metadata.putObjectMD(mpuBucketName, partKey, omVal, mdParams, log, - err => { + }, + // Store in data backend. + ( + destinationBucket, + objectLocationConstraint, + cipherBundle, + partKey, + prevObjectSize, + oldLocations, + partInfo, + splitter, + next, + ) => { + // NOTE: set oldLocations to null so we do not batchDelete for now + if (partInfo && constants.skipBatchDeleteBackends[partInfo.dataStoreType]) { + // skip to storing metadata + return next( + null, + destinationBucket, + partInfo, + partInfo.dataStoreETag, + cipherBundle, + partKey, + prevObjectSize, + null, + objectLocationConstraint, + splitter, + ); + } + const objectContext = { + bucketName, + owner: canonicalID, + namespace: request.namespace, + objectKey, + partNumber: _getPaddedPartNumber(partNumber), + uploadId, + }; + const backendInfo = new BackendInfo(config, objectLocationConstraint); + return dataStore( + objectContext, + cipherBundle, + request, + size, + streamingV4Params, + backendInfo, + log, + (err, dataGetInfo, hexDigest) => { + if (err) { + return next(err, destinationBucket); + } + return next( + null, + destinationBucket, + dataGetInfo, + hexDigest, + cipherBundle, + partKey, + prevObjectSize, + oldLocations, + objectLocationConstraint, + splitter, + ); + }, + ); + }, + // Store data locations in metadata and delete any overwritten + // data if completeMPU hasn't been initiated yet. + ( + destinationBucket, + dataGetInfo, + hexDigest, + cipherBundle, + partKey, + prevObjectSize, + oldLocations, + objectLocationConstraint, + splitter, + next, + ) => { + // Use an array to be consistent with objectPutCopyPart where there + // could be multiple locations. + const partLocations = [dataGetInfo]; + const sseHeaders = {}; + if (cipherBundle) { + const { algorithm, masterKeyId, cryptoScheme, cipheredDataKey } = cipherBundle; + partLocations[0].sseAlgorithm = algorithm; + partLocations[0].sseMasterKeyId = masterKeyId; + partLocations[0].sseCryptoScheme = cryptoScheme; + partLocations[0].sseCipheredDataKey = cipheredDataKey; + sseHeaders.algo = algorithm; + sseHeaders.kmsKey = masterKeyId; + } + const omVal = { + // back to Version 3 since number-subparts is not needed + 'md-model-version': 3, + partLocations, + key: partKey, + 'last-modified': new Date().toJSON(), + 'content-md5': hexDigest, + 'content-length': size, + 'owner-id': destinationBucket.getOwner(), + }; + const mdParams = { overheadField: constants.overheadField }; + return metadata.putObjectMD(mpuBucketName, partKey, omVal, mdParams, log, err => { if (err) { log.error('error putting object in mpu bucket', { error: err, @@ -364,100 +429,144 @@ function objectPutPart(authInfo, request, streamingV4Params, log, }); return next(err, destinationBucket); } - return next(null, partLocations, oldLocations, objectLocationConstraint, - destinationBucket, hexDigest, sseHeaders, prevObjectSize, splitter); + return next( + null, + partLocations, + oldLocations, + objectLocationConstraint, + destinationBucket, + hexDigest, + sseHeaders, + prevObjectSize, + splitter, + ); }); - }, - (partLocations, oldLocations, objectLocationConstraint, destinationBucket, - hexDigest, sseHeaders, prevObjectSize, splitter, next) => { - if (!oldLocations) { - return next(null, oldLocations, objectLocationConstraint, - destinationBucket, hexDigest, sseHeaders, prevObjectSize); - } - return services.isCompleteMPUInProgress({ - bucketName, - objectKey, - uploadId, + }, + ( + partLocations, + oldLocations, + objectLocationConstraint, + destinationBucket, + hexDigest, + sseHeaders, + prevObjectSize, splitter, - }, log, (err, completeInProgress) => { - if (err) { - return next(err, destinationBucket); + next, + ) => { + if (!oldLocations) { + return next( + null, + oldLocations, + objectLocationConstraint, + destinationBucket, + hexDigest, + sseHeaders, + prevObjectSize, + ); } - let oldLocationsToDelete = oldLocations; - // Prevent deletion of old data if a completeMPU - // is already in progress because then there is no - // guarantee that the old location will not be the - // committed one. - if (completeInProgress) { - log.warn('not deleting old locations because CompleteMPU is in progress', { - method: 'objectPutPart::metadata.getObjectMD', + return services.isCompleteMPUInProgress( + { bucketName, objectKey, uploadId, - partLocations, - oldLocations, - }); - oldLocationsToDelete = null; - } - return next(null, oldLocationsToDelete, objectLocationConstraint, - destinationBucket, hexDigest, sseHeaders, prevObjectSize); - }); - }, - // Clean up any old data now that new metadata (with new - // data locations) has been stored. - (oldLocationsToDelete, objectLocationConstraint, destinationBucket, hexDigest, - sseHeaders, prevObjectSize, next) => { - if (oldLocationsToDelete) { - log.trace('overwriting mpu part, deleting data'); - return data.batchDelete(oldLocationsToDelete, request.method, - objectLocationConstraint, log, err => { + splitter, + }, + log, + (err, completeInProgress) => { if (err) { - // if error, log the error and move on as it is not - // relevant to the client as the client's - // object already succeeded putting data, metadata - log.error('error deleting existing data', - { error: err }); + return next(err, destinationBucket); + } + let oldLocationsToDelete = oldLocations; + // Prevent deletion of old data if a completeMPU + // is already in progress because then there is no + // guarantee that the old location will not be the + // committed one. + if (completeInProgress) { + log.warn('not deleting old locations because CompleteMPU is in progress', { + method: 'objectPutPart::metadata.getObjectMD', + bucketName, + objectKey, + uploadId, + partLocations, + oldLocations, + }); + oldLocationsToDelete = null; } - return next(null, destinationBucket, hexDigest, - sseHeaders, prevObjectSize); - }); + return next( + null, + oldLocationsToDelete, + objectLocationConstraint, + destinationBucket, + hexDigest, + sseHeaders, + prevObjectSize, + ); + }, + ); + }, + // Clean up any old data now that new metadata (with new + // data locations) has been stored. + ( + oldLocationsToDelete, + objectLocationConstraint, + destinationBucket, + hexDigest, + sseHeaders, + prevObjectSize, + next, + ) => { + if (oldLocationsToDelete) { + log.trace('overwriting mpu part, deleting data'); + return data.batchDelete( + oldLocationsToDelete, + request.method, + objectLocationConstraint, + log, + err => { + if (err) { + // if error, log the error and move on as it is not + // relevant to the client as the client's + // object already succeeded putting data, metadata + log.error('error deleting existing data', { error: err }); + } + return next(null, destinationBucket, hexDigest, sseHeaders, prevObjectSize); + }, + ); + } + return next(null, destinationBucket, hexDigest, sseHeaders, prevObjectSize); + }, + ], + (err, destinationBucket, hexDigest, sseHeaders, prevObjectSize) => { + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, destinationBucket); + // eslint-disable-next-line no-param-reassign + request.actionImplicitDenies = originalIdentityAuthzResults; + if (sseHeaders) { + setSSEHeaders(corsHeaders, sseHeaders.algo, sseHeaders.kmsKey); } - return next(null, destinationBucket, hexDigest, - sseHeaders, prevObjectSize); - }, - ], (err, destinationBucket, hexDigest, sseHeaders, prevObjectSize) => { - const corsHeaders = collectCorsHeaders(request.headers.origin, - request.method, destinationBucket); - // eslint-disable-next-line no-param-reassign - request.actionImplicitDenies = originalIdentityAuthzResults; - if (sseHeaders) { - setSSEHeaders(corsHeaders, sseHeaders.algo, sseHeaders.kmsKey); - } - if (err) { - if (err === skipError) { - return cb(null, hexDigest, corsHeaders); + if (err) { + if (err === skipError) { + return cb(null, hexDigest, corsHeaders); + } + log.error('error in object put part (upload part)', { + error: err, + method: 'objectPutPart', + }); + monitoring.promMetrics('PUT', bucketName, err.code, 'putObjectPart'); + return cb(err, null, corsHeaders); } - log.error('error in object put part (upload part)', { - error: err, - method: 'objectPutPart', + pushMetric('uploadPart', log, { + authInfo, + canonicalID: destinationBucket.getOwner(), + bucket: bucketName, + keys: [objectKey], + newByteLength: size, + oldByteLength: prevObjectSize, + location: destinationBucket.getLocationConstraint(), }); - monitoring.promMetrics('PUT', bucketName, err.code, - 'putObjectPart'); - return cb(err, null, corsHeaders); - } - pushMetric('uploadPart', log, { - authInfo, - canonicalID: destinationBucket.getOwner(), - bucket: bucketName, - keys: [objectKey], - newByteLength: size, - oldByteLength: prevObjectSize, - location: destinationBucket.getLocationConstraint(), - }); - monitoring.promMetrics('PUT', bucketName, - '200', 'putObjectPart', size, prevObjectSize); - return cb(null, hexDigest, corsHeaders); - }); + monitoring.promMetrics('PUT', bucketName, '200', 'putObjectPart', size, prevObjectSize); + return cb(null, hexDigest, corsHeaders); + }, + ); } module.exports = objectPutPart; diff --git a/lib/api/objectPutRetention.js b/lib/api/objectPutRetention.js index 6a7a2c8441..b8182646e9 100644 --- a/lib/api/objectPutRetention.js +++ b/lib/api/objectPutRetention.js @@ -1,10 +1,12 @@ const async = require('async'); const { errors, errorInstances, s3middleware } = require('arsenal'); -const { decodeVersionId, getVersionIdResHeader, getVersionSpecificMetadataOptions } = - require('./apiUtils/object/versioning'); -const { ObjectLockInfo, hasGovernanceBypassHeader } = - require('./apiUtils/object/objectLockHelpers'); +const { + decodeVersionId, + getVersionIdResHeader, + getVersionSpecificMetadataOptions, +} = require('./apiUtils/object/versioning'); +const { ObjectLockInfo, hasGovernanceBypassHeader } = require('./apiUtils/object/objectLockHelpers'); const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUtils'); const { pushMetric } = require('../utapi/utilities'); const getReplicationInfo = require('./apiUtils/object/getReplicationInfo'); @@ -50,99 +52,106 @@ function objectPutRetention(authInfo, request, log, callback) { const hasGovernanceBypass = hasGovernanceBypassHeader(request.headers); - return async.waterfall([ - next => { - log.trace('parsing retention information'); - parseRetentionXml(request.post, log, - (err, retentionInfo) => { + return async.waterfall( + [ + next => { + log.trace('parsing retention information'); + parseRetentionXml(request.post, log, (err, retentionInfo) => { if (err) { - log.trace('error parsing retention information', - { error: err }); + log.trace('error parsing retention information', { error: err }); return next(err); } - const remainingDays = Math.ceil( - (new Date(retentionInfo.date) - Date.now()) / (1000 * 3600 * 24)); + const remainingDays = Math.ceil((new Date(retentionInfo.date) - Date.now()) / (1000 * 3600 * 24)); metadataValParams.request.objectLockRetentionDays = remainingDays; return next(null, retentionInfo); }); - }, - (retentionInfo, next) => standardMetadataValidateBucketAndObj(metadataValParams, - request.actionImplicitDenies, log, (err, bucket, objectMD) => { - if (err) { - log.trace('request authorization failed', - { method: 'objectPutRetention', error: err }); - return next(err); - } - if (!objectMD) { - const err = reqVersionId ? errors.NoSuchVersion : - errors.NoSuchKey; - log.trace('error no object metadata found', - { method: 'objectPutRetention', error: err }); - return next(err, bucket); - } - if (objectMD.isDeleteMarker) { - log.trace('version is a delete marker', - { method: 'objectPutRetention' }); - return next(errors.MethodNotAllowed, bucket); - } - if (!bucket.isObjectLockEnabled()) { - log.trace('object lock not enabled on bucket', - { method: 'objectPutRetention' }); - return next(errorInstances.InvalidRequest.customizeDescription( - 'Bucket is missing Object Lock Configuration' - ), bucket); - } - return next(null, bucket, retentionInfo, objectMD); - }), - (bucket, retentionInfo, objectMD, next) => { - const objLockInfo = new ObjectLockInfo({ - mode: objectMD.retentionMode, - date: objectMD.retentionDate, - legalHold: objectMD.legalHold, - }); + }, + (retentionInfo, next) => + standardMetadataValidateBucketAndObj( + metadataValParams, + request.actionImplicitDenies, + log, + (err, bucket, objectMD) => { + if (err) { + log.trace('request authorization failed', { method: 'objectPutRetention', error: err }); + return next(err); + } + if (!objectMD) { + const err = reqVersionId ? errors.NoSuchVersion : errors.NoSuchKey; + log.trace('error no object metadata found', { method: 'objectPutRetention', error: err }); + return next(err, bucket); + } + if (objectMD.isDeleteMarker) { + log.trace('version is a delete marker', { method: 'objectPutRetention' }); + return next(errors.MethodNotAllowed, bucket); + } + if (!bucket.isObjectLockEnabled()) { + log.trace('object lock not enabled on bucket', { method: 'objectPutRetention' }); + return next( + errorInstances.InvalidRequest.customizeDescription( + 'Bucket is missing Object Lock Configuration', + ), + bucket, + ); + } + return next(null, bucket, retentionInfo, objectMD); + }, + ), + (bucket, retentionInfo, objectMD, next) => { + const objLockInfo = new ObjectLockInfo({ + mode: objectMD.retentionMode, + date: objectMD.retentionDate, + legalHold: objectMD.legalHold, + }); - if (!objLockInfo.canModifyPolicy(retentionInfo, hasGovernanceBypass)) { - return next(errors.AccessDenied, bucket); - } + if (!objLockInfo.canModifyPolicy(retentionInfo, hasGovernanceBypass)) { + return next(errors.AccessDenied, bucket); + } - return next(null, bucket, retentionInfo, objectMD); - }, - (bucket, retentionInfo, objectMD, next) => { - /* eslint-disable no-param-reassign */ - objectMD.retentionMode = retentionInfo.mode; - objectMD.retentionDate = retentionInfo.date; - const params = getVersionSpecificMetadataOptions(objectMD, config.nullVersionCompatMode); - const replicationInfo = getReplicationInfo(config, - objectKey, bucket, true, 0, REPLICATION_ACTION, objectMD); - if (replicationInfo) { - objectMD.replicationInfo = Object.assign({}, - objectMD.replicationInfo, replicationInfo); + return next(null, bucket, retentionInfo, objectMD); + }, + (bucket, retentionInfo, objectMD, next) => { + /* eslint-disable no-param-reassign */ + objectMD.retentionMode = retentionInfo.mode; + objectMD.retentionDate = retentionInfo.date; + const params = getVersionSpecificMetadataOptions(objectMD, config.nullVersionCompatMode); + const replicationInfo = getReplicationInfo( + config, + objectKey, + bucket, + true, + 0, + REPLICATION_ACTION, + objectMD, + ); + if (replicationInfo) { + objectMD.replicationInfo = Object.assign({}, objectMD.replicationInfo, replicationInfo); + } + objectMD.originOp = 's3:ObjectRetention:Put'; + /* eslint-enable no-param-reassign */ + metadata.putObjectMD(bucket.getName(), objectKey, objectMD, params, log, err => + next(err, bucket, objectMD), + ); + }, + ], + (err, bucket, objectMD) => { + const additionalResHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + if (err) { + log.trace('error processing request', { error: err, method: 'objectPutRetention' }); + } else { + pushMetric('putObjectRetention', log, { + authInfo, + bucket: bucketName, + keys: [objectKey], + versionId: objectMD ? objectMD.versionId : undefined, + location: objectMD ? objectMD.dataStoreName : undefined, + }); + const verCfg = bucket.getVersioningConfiguration(); + additionalResHeaders['x-amz-version-id'] = getVersionIdResHeader(verCfg, objectMD); } - objectMD.originOp = 's3:ObjectRetention:Put'; - /* eslint-enable no-param-reassign */ - metadata.putObjectMD(bucket.getName(), objectKey, objectMD, params, - log, err => next(err, bucket, objectMD)); + return callback(err, additionalResHeaders); }, - ], (err, bucket, objectMD) => { - const additionalResHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); - if (err) { - log.trace('error processing request', - { error: err, method: 'objectPutRetention' }); - } else { - pushMetric('putObjectRetention', log, { - authInfo, - bucket: bucketName, - keys: [objectKey], - versionId: objectMD ? objectMD.versionId : undefined, - location: objectMD ? objectMD.dataStoreName : undefined, - }); - const verCfg = bucket.getVersioningConfiguration(); - additionalResHeaders['x-amz-version-id'] = - getVersionIdResHeader(verCfg, objectMD); - } - return callback(err, additionalResHeaders); - }); + ); } module.exports = objectPutRetention; diff --git a/lib/api/objectPutTagging.js b/lib/api/objectPutTagging.js index ef23dcf64d..85bb652787 100644 --- a/lib/api/objectPutTagging.js +++ b/lib/api/objectPutTagging.js @@ -1,8 +1,11 @@ const async = require('async'); const { errors, s3middleware } = require('arsenal'); -const { decodeVersionId, getVersionIdResHeader, getVersionSpecificMetadataOptions } = - require('./apiUtils/object/versioning'); +const { + decodeVersionId, + getVersionIdResHeader, + getVersionSpecificMetadataOptions, +} = require('./apiUtils/object/versioning'); const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUtils'); const { pushMetric } = require('../utapi/utilities'); @@ -47,80 +50,85 @@ function objectPutTagging(authInfo, request, log, callback) { request, }; - return async.waterfall([ - next => standardMetadataValidateBucketAndObj(metadataValParams, request.actionImplicitDenies, log, - (err, bucket, objectMD) => { - if (err) { - log.trace('request authorization failed', - { method: 'objectPutTagging', error: err }); - return next(err); - } - if (!objectMD) { - const err = reqVersionId ? errors.NoSuchVersion : - errors.NoSuchKey; - log.trace('error no object metadata found', - { method: 'objectPutTagging', error: err }); - return next(err, bucket); - } - if (objectMD.isDeleteMarker) { - log.trace('version is a delete marker', - { method: 'objectPutTagging' }); - // FIXME we should return a `x-amz-delete-marker: true` header, - // see S3C-7592 - return next(errors.MethodNotAllowed, bucket); - } - return next(null, bucket, objectMD); - }), - (bucket, objectMD, next) => { - log.trace('parsing tag(s)'); - parseTagXml(request.post, log, (err, tags) => - next(err, bucket, tags, objectMD)); - }, - (bucket, tags, objectMD, next) => { - // eslint-disable-next-line no-param-reassign - objectMD.tags = tags; - const params = getVersionSpecificMetadataOptions(objectMD, config.nullVersionCompatMode); - const replicationInfo = getReplicationInfo(config, - objectKey, bucket, true, 0, REPLICATION_ACTION, objectMD); - if (replicationInfo) { + return async.waterfall( + [ + next => + standardMetadataValidateBucketAndObj( + metadataValParams, + request.actionImplicitDenies, + log, + (err, bucket, objectMD) => { + if (err) { + log.trace('request authorization failed', { method: 'objectPutTagging', error: err }); + return next(err); + } + if (!objectMD) { + const err = reqVersionId ? errors.NoSuchVersion : errors.NoSuchKey; + log.trace('error no object metadata found', { method: 'objectPutTagging', error: err }); + return next(err, bucket); + } + if (objectMD.isDeleteMarker) { + log.trace('version is a delete marker', { method: 'objectPutTagging' }); + // FIXME we should return a `x-amz-delete-marker: true` header, + // see S3C-7592 + return next(errors.MethodNotAllowed, bucket); + } + return next(null, bucket, objectMD); + }, + ), + (bucket, objectMD, next) => { + log.trace('parsing tag(s)'); + parseTagXml(request.post, log, (err, tags) => next(err, bucket, tags, objectMD)); + }, + (bucket, tags, objectMD, next) => { + // eslint-disable-next-line no-param-reassign + objectMD.tags = tags; + const params = getVersionSpecificMetadataOptions(objectMD, config.nullVersionCompatMode); + const replicationInfo = getReplicationInfo( + config, + objectKey, + bucket, + true, + 0, + REPLICATION_ACTION, + objectMD, + ); + if (replicationInfo) { + // eslint-disable-next-line no-param-reassign + objectMD.replicationInfo = Object.assign({}, objectMD.replicationInfo, replicationInfo); + } // eslint-disable-next-line no-param-reassign - objectMD.replicationInfo = Object.assign({}, - objectMD.replicationInfo, replicationInfo); + objectMD.originOp = 's3:ObjectTagging:Put'; + metadata.putObjectMD(bucket.getName(), objectKey, objectMD, params, log, err => + next(err, bucket, objectMD), + ); + }, + (bucket, objectMD, next) => + // if external backend handles tagging + data.objectTagging('Put', objectKey, bucket.getName(), objectMD, log, err => + next(err, bucket, objectMD), + ), + ], + (err, bucket, objectMD) => { + const additionalResHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + if (err) { + log.trace('error processing request', { error: err, method: 'objectPutTagging' }); + monitoring.promMetrics('PUT', bucketName, err.code, 'putObjectTagging'); + } else { + pushMetric('putObjectTagging', log, { + authInfo, + bucket: bucketName, + keys: [objectKey], + versionId: objectMD ? objectMD.versionId : undefined, + location: objectMD ? objectMD.dataStoreName : undefined, + }); + monitoring.promMetrics('PUT', bucketName, '200', 'putObjectTagging'); + const verCfg = bucket.getVersioningConfiguration(); + additionalResHeaders['x-amz-version-id'] = getVersionIdResHeader(verCfg, objectMD); } - // eslint-disable-next-line no-param-reassign - objectMD.originOp = 's3:ObjectTagging:Put'; - metadata.putObjectMD(bucket.getName(), objectKey, objectMD, params, - log, err => - next(err, bucket, objectMD)); + return callback(err, additionalResHeaders); }, - (bucket, objectMD, next) => - // if external backend handles tagging - data.objectTagging('Put', objectKey, bucket.getName(), objectMD, - log, err => next(err, bucket, objectMD)), - ], (err, bucket, objectMD) => { - const additionalResHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); - if (err) { - log.trace('error processing request', { error: err, - method: 'objectPutTagging' }); - monitoring.promMetrics('PUT', bucketName, err.code, - 'putObjectTagging'); - } else { - pushMetric('putObjectTagging', log, { - authInfo, - bucket: bucketName, - keys: [objectKey], - versionId: objectMD ? objectMD.versionId : undefined, - location: objectMD ? objectMD.dataStoreName : undefined, - }); - monitoring.promMetrics( - 'PUT', bucketName, '200', 'putObjectTagging'); - const verCfg = bucket.getVersioningConfiguration(); - additionalResHeaders['x-amz-version-id'] = - getVersionIdResHeader(verCfg, objectMD); - } - return callback(err, additionalResHeaders); - }); + ); } module.exports = objectPutTagging; diff --git a/lib/api/objectRestore.js b/lib/api/objectRestore.js index be0eb48389..f96234dc4e 100644 --- a/lib/api/objectRestore.js +++ b/lib/api/objectRestore.js @@ -22,8 +22,7 @@ const sdtObjectRestore = require('./apiUtils/object/objectRestore'); * @return {undefined} */ function objectRestore(userInfo, request, log, callback) { - return sdtObjectRestore(metadata, metadataUtils, userInfo, request, - log, callback); + return sdtObjectRestore(metadata, metadataUtils, userInfo, request, log, callback); } module.exports = objectRestore; diff --git a/lib/api/serviceGet.js b/lib/api/serviceGet.js index f81169b3f9..79b980b8c0 100644 --- a/lib/api/serviceGet.js +++ b/lib/api/serviceGet.js @@ -34,9 +34,8 @@ function generateXml(xml, owner, userBuckets, splitter) { xml.push( '', `${key}`, - `${bucket.value.creationDate}` + - '', - '' + `${bucket.value.creationDate}` + '', + '', ); }); xml.push(''); @@ -56,38 +55,32 @@ function serviceGet(authInfo, request, log, callback) { if (authInfo.isRequesterPublicUser()) { log.debug('operation not available for public user'); - monitoring.promMetrics( - 'GET', request.bucketName, 403, 'getService'); + monitoring.promMetrics('GET', request.bucketName, 403, 'getService'); return callback(errors.AccessDenied); } const xml = []; const canonicalId = authInfo.getCanonicalID(); xml.push( '', - '', + '', '', `${canonicalId}`, - `${authInfo.getAccountDisplayName()}` + - '', + `${authInfo.getAccountDisplayName()}` + '', '', - '' + '', ); - return services.getService(authInfo, request, log, constants.splitter, - (err, userBuckets, splitter) => { - if (err) { - monitoring.promMetrics( - 'GET', userBuckets, err.code, 'getService'); - return callback(err); - } - // TODO push metric for serviceGet - // pushMetric('getService', log, { - // bucket: bucketName, - // }); - monitoring.promMetrics('GET', userBuckets, '200', 'getService'); - return callback(null, generateXml(xml, canonicalId, userBuckets, - splitter)); - }); + return services.getService(authInfo, request, log, constants.splitter, (err, userBuckets, splitter) => { + if (err) { + monitoring.promMetrics('GET', userBuckets, err.code, 'getService'); + return callback(err); + } + // TODO push metric for serviceGet + // pushMetric('getService', log, { + // bucket: bucketName, + // }); + monitoring.promMetrics('GET', userBuckets, '200', 'getService'); + return callback(null, generateXml(xml, canonicalId, userBuckets, splitter)); + }); } module.exports = serviceGet; diff --git a/lib/api/website.js b/lib/api/website.js index 801d92f708..ed23ba3e5e 100644 --- a/lib/api/website.js +++ b/lib/api/website.js @@ -5,10 +5,12 @@ const collectCorsHeaders = require('../utilities/collectCorsHeaders'); const constants = require('../../constants'); const metadata = require('../metadata/wrapper'); const bucketShield = require('./apiUtils/bucket/bucketShield'); -const { appendWebsiteIndexDocument, findRoutingRule, extractRedirectInfo } = - require('./apiUtils/object/websiteServing'); -const { isObjAuthorized, isBucketAuthorized } = - require('./apiUtils/authorization/permissionChecks'); +const { + appendWebsiteIndexDocument, + findRoutingRule, + extractRedirectInfo, +} = require('./apiUtils/object/websiteServing'); +const { isObjAuthorized, isBucketAuthorized } = require('./apiUtils/authorization/permissionChecks'); const collectResponseHeaders = require('../utilities/collectResponseHeaders'); const { pushMetric } = require('../utapi/utilities'); const monitoring = require('../utilities/monitoringHandler'); @@ -27,81 +29,74 @@ const monitoring = require('../utilities/monitoringHandler'); * @param {function} callback - callback to function in route * @return {undefined} */ -function _errorActions(err, errorDocument, routingRules, - bucket, objectKey, corsHeaders, request, log, callback) { +function _errorActions(err, errorDocument, routingRules, bucket, objectKey, corsHeaders, request, log, callback) { const bucketName = bucket.getName(); - const errRoutingRule = findRoutingRule(routingRules, - objectKey, err.code); + const errRoutingRule = findRoutingRule(routingRules, objectKey, err.code); if (errRoutingRule) { // route will redirect const action = request.method === 'HEAD' ? 'headObject' : 'getObject'; - monitoring.promMetrics( - request.method, bucketName, err.code, action); - return callback(err, false, null, corsHeaders, errRoutingRule, - objectKey); + monitoring.promMetrics(request.method, bucketName, err.code, action); + return callback(err, false, null, corsHeaders, errRoutingRule, objectKey); } if (request.method === 'HEAD') { - monitoring.promMetrics( - 'HEAD', bucketName, err.code, 'headObject'); + monitoring.promMetrics('HEAD', bucketName, err.code, 'headObject'); return callback(err, false, null, corsHeaders); } if (errorDocument) { - return metadata.getObjectMD(bucketName, errorDocument, {}, log, - (errObjErr, errObjMD) => { - if (errObjErr) { - // error retrieving error document so return original error - // and set boolean of error retrieving user's error document - // to true - monitoring.promMetrics( - 'GET', bucketName, err.code, 'getObject'); - return callback(err, true, null, corsHeaders); - } - // return the default error message if the object is private - // rather than sending a stored error file - // eslint-disable-next-line no-param-reassign - request.objectKey = errorDocument; - if (!isObjAuthorized(bucket, errObjMD, request.apiMethods || 'objectGet', - constants.publicId, null, log, request, request.actionImplicitDenies, true)) { - log.trace('errorObj not authorized', { error: err }); - monitoring.promMetrics( - 'GET', bucketName, err.code, 'getObject'); - return callback(err, true, null, corsHeaders); - } - const dataLocator = errObjMD.location; - if (errObjMD['x-amz-server-side-encryption']) { - for (let i = 0; i < dataLocator.length; i++) { - dataLocator[i].masterKeyId = - errObjMD['x-amz-server-side-encryption-aws-' + - 'kms-key-id']; - dataLocator[i].algorithm = - errObjMD['x-amz-server-side-encryption']; - } + return metadata.getObjectMD(bucketName, errorDocument, {}, log, (errObjErr, errObjMD) => { + if (errObjErr) { + // error retrieving error document so return original error + // and set boolean of error retrieving user's error document + // to true + monitoring.promMetrics('GET', bucketName, err.code, 'getObject'); + return callback(err, true, null, corsHeaders); + } + // return the default error message if the object is private + // rather than sending a stored error file + // eslint-disable-next-line no-param-reassign + request.objectKey = errorDocument; + if ( + !isObjAuthorized( + bucket, + errObjMD, + request.apiMethods || 'objectGet', + constants.publicId, + null, + log, + request, + request.actionImplicitDenies, + true, + ) + ) { + log.trace('errorObj not authorized', { error: err }); + monitoring.promMetrics('GET', bucketName, err.code, 'getObject'); + return callback(err, true, null, corsHeaders); + } + const dataLocator = errObjMD.location; + if (errObjMD['x-amz-server-side-encryption']) { + for (let i = 0; i < dataLocator.length; i++) { + dataLocator[i].masterKeyId = errObjMD['x-amz-server-side-encryption-aws-' + 'kms-key-id']; + dataLocator[i].algorithm = errObjMD['x-amz-server-side-encryption']; } + } - if (errObjMD['x-amz-website-redirect-location']) { - const redirectLocation = - errObjMD['x-amz-website-redirect-location']; - const redirectInfo = { withError: true, - location: redirectLocation }; - log.trace('redirecting to x-amz-website-redirect-location', - { location: redirectLocation }); - return callback(err, false, dataLocator, corsHeaders, - redirectInfo, ''); - } + if (errObjMD['x-amz-website-redirect-location']) { + const redirectLocation = errObjMD['x-amz-website-redirect-location']; + const redirectInfo = { withError: true, location: redirectLocation }; + log.trace('redirecting to x-amz-website-redirect-location', { location: redirectLocation }); + return callback(err, false, dataLocator, corsHeaders, redirectInfo, ''); + } - const responseMetaHeaders = collectResponseHeaders(errObjMD, - corsHeaders); - pushMetric('getObject', log, { - bucket: bucketName, - newByteLength: responseMetaHeaders['Content-Length'], - }); - monitoring.promMetrics( - 'GET', bucketName, err.code, 'getObject'); - return callback(err, false, dataLocator, responseMetaHeaders); + const responseMetaHeaders = collectResponseHeaders(errObjMD, corsHeaders); + pushMetric('getObject', log, { + bucket: bucketName, + newByteLength: responseMetaHeaders['Content-Length'], }); + monitoring.promMetrics('GET', bucketName, err.code, 'getObject'); + return callback(err, false, dataLocator, responseMetaHeaders); + }); } - monitoring.promMetrics( - 'GET', bucketName, err.code, 'getObject'); + monitoring.promMetrics('GET', bucketName, err.code, 'getObject'); return callback(err, false, null, corsHeaders); } @@ -120,8 +115,7 @@ function capitalize(str) { * @returns {function} HEAD callback with GET signature */ function callbackGetToHead(callback) { - return (err, userErrorPageFailure, dataGetInfo, - resMetaHeaders, redirectInfo, key) => + return (err, userErrorPageFailure, dataGetInfo, resMetaHeaders, redirectInfo, key) => callback(err, resMetaHeaders, redirectInfo, key); } @@ -147,26 +141,21 @@ function website(request, log, callback) { return metadata.getBucket(bucketName, log, (err, bucket) => { if (err) { log.trace('error retrieving bucket metadata', { error: err }); - monitoring.promMetrics( - request.method, bucketName, err.code, action); + monitoring.promMetrics(request.method, bucketName, err.code, action); return callback(err, false); } if (bucketShield(bucket, `object${methodCapitalized}`)) { log.trace('bucket in transient/deleted state so shielding'); - monitoring.promMetrics( - request.method, bucketName, 404, action); + monitoring.promMetrics(request.method, bucketName, 404, action); return callback(errors.NoSuchBucket, false); } - const corsHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); // bucket ACL's do not matter for website head since it is always the // head of an object. object ACL's are what matter const websiteConfig = bucket.getWebsiteConfiguration(); if (!websiteConfig) { - monitoring.promMetrics( - request.method, bucketName, 404, action); - return callback(errors.NoSuchWebsiteConfiguration, false, null, - corsHeaders); + monitoring.promMetrics(request.method, bucketName, 404, action); + return callback(errors.NoSuchWebsiteConfiguration, false, null, corsHeaders); } // any errors above would be our own created generic error html // if have a website config, error going forward would be user's @@ -174,8 +163,7 @@ function website(request, log, callback) { // handle redirect all if (websiteConfig.getRedirectAllRequestsTo()) { - return callback(null, false, null, corsHeaders, - websiteConfig.getRedirectAllRequestsTo(), reqObjectKey); + return callback(null, false, null, corsHeaders, websiteConfig.getRedirectAllRequestsTo(), reqObjectKey); } // check whether need to redirect based on key @@ -185,8 +173,7 @@ function website(request, log, callback) { if (keyRoutingRule) { // TODO: optimize by not rerouting if only routing // rule is to change out key - return callback(null, false, null, corsHeaders, - keyRoutingRule, reqObjectKey); + return callback(null, false, null, corsHeaders, keyRoutingRule, reqObjectKey); } appendWebsiteIndexDocument(request, websiteConfig.getIndexDocument()); @@ -203,107 +190,134 @@ function website(request, log, callback) { function runWebsite(originalError) { // get object metadata and check authorization and header // validation - return metadata.getObjectMD(bucketName, request.objectKey, {}, log, - (err, objMD) => { - // Note: In case of error, we intentionally send the original - // object key to _errorActions as in case of a redirect, we do - // not want to append index key to redirect location - if (err) { - log.trace('error retrieving object metadata', - { error: err }); - let returnErr = err; - const bucketAuthorized = isBucketAuthorized(bucket, request.apiMethods || 'bucketGet', - constants.publicId, null, log, request, request.actionImplicitDenies, true); - // if index object does not exist and bucket is private AWS - // returns 403 - AccessDenied error. - if (err.is.NoSuchKey && !bucketAuthorized) { - returnErr = errors.AccessDenied; - } - - // Check if key is a folder containing index for redirect 302 - // https://docs.aws.amazon.com/AmazonS3/latest/userguide/IndexDocumentSupport.html - if (!originalError && reqObjectKey && !reqObjectKey.endsWith('/')) { - appendWebsiteIndexDocument(request, websiteConfig.getIndexDocument(), true); - // propagate returnErr as originalError to be used if index is not found - return runWebsite(returnErr); - } - - return _errorActions(originalError || returnErr, - websiteConfig.getErrorDocument(), routingRules, - bucket, reqObjectKey, corsHeaders, request, log, - callback); - } - if (!isObjAuthorized(bucket, objMD, request.apiMethods || 'objectGet', - constants.publicId, null, log, request, request.actionImplicitDenies, true)) { - const err = errors.AccessDenied; - log.trace('request not authorized', { error: err }); - return _errorActions(err, websiteConfig.getErrorDocument(), - routingRules, bucket, - reqObjectKey, corsHeaders, request, log, callback); + return metadata.getObjectMD(bucketName, request.objectKey, {}, log, (err, objMD) => { + // Note: In case of error, we intentionally send the original + // object key to _errorActions as in case of a redirect, we do + // not want to append index key to redirect location + if (err) { + log.trace('error retrieving object metadata', { error: err }); + let returnErr = err; + const bucketAuthorized = isBucketAuthorized( + bucket, + request.apiMethods || 'bucketGet', + constants.publicId, + null, + log, + request, + request.actionImplicitDenies, + true, + ); + // if index object does not exist and bucket is private AWS + // returns 403 - AccessDenied error. + if (err.is.NoSuchKey && !bucketAuthorized) { + returnErr = errors.AccessDenied; } - // access granted to index document, needs a redirect 302 - // to the original key with trailing / - if (originalError) { - const redirectInfo = { withError: true, - location: `/${reqObjectKey}/` }; - return callback(errors.Found, false, null, corsHeaders, - redirectInfo, ''); + // Check if key is a folder containing index for redirect 302 + // https://docs.aws.amazon.com/AmazonS3/latest/userguide/IndexDocumentSupport.html + if (!originalError && reqObjectKey && !reqObjectKey.endsWith('/')) { + appendWebsiteIndexDocument(request, websiteConfig.getIndexDocument(), true); + // propagate returnErr as originalError to be used if index is not found + return runWebsite(returnErr); } - const headerValResult = validateHeaders(request.headers, - objMD['last-modified'], objMD['content-md5']); - if (headerValResult.error) { - const err = headerValResult.error; - log.trace('header validation error', { error: err }); - return _errorActions(err, websiteConfig.getErrorDocument(), - routingRules, bucket, reqObjectKey, - corsHeaders, request, log, callback); - } - // check if object to serve has website redirect header - // Note: AWS prioritizes website configuration rules over - // object key's website redirect header, so we make the - // check at the end. - if (objMD['x-amz-website-redirect-location']) { - const redirectLocation = - objMD['x-amz-website-redirect-location']; - const redirectInfo = - extractRedirectInfo(redirectLocation); - log.trace('redirecting to x-amz-website-redirect-location', - { location: redirectLocation }); - return callback(null, false, null, corsHeaders, - redirectInfo, ''); - } - // got obj metadata, authorized and headers validated, - // good to go - const responseMetaHeaders = collectResponseHeaders(objMD, - corsHeaders); + return _errorActions( + originalError || returnErr, + websiteConfig.getErrorDocument(), + routingRules, + bucket, + reqObjectKey, + corsHeaders, + request, + log, + callback, + ); + } + if ( + !isObjAuthorized( + bucket, + objMD, + request.apiMethods || 'objectGet', + constants.publicId, + null, + log, + request, + request.actionImplicitDenies, + true, + ) + ) { + const err = errors.AccessDenied; + log.trace('request not authorized', { error: err }); + return _errorActions( + err, + websiteConfig.getErrorDocument(), + routingRules, + bucket, + reqObjectKey, + corsHeaders, + request, + log, + callback, + ); + } - if (request.method === 'HEAD') { - pushMetric('headObject', log, { bucket: bucketName }); - monitoring.promMetrics('HEAD', - bucketName, '200', 'headObject'); - return callback(null, false, null, responseMetaHeaders); - } + // access granted to index document, needs a redirect 302 + // to the original key with trailing / + if (originalError) { + const redirectInfo = { withError: true, location: `/${reqObjectKey}/` }; + return callback(errors.Found, false, null, corsHeaders, redirectInfo, ''); + } + + const headerValResult = validateHeaders(request.headers, objMD['last-modified'], objMD['content-md5']); + if (headerValResult.error) { + const err = headerValResult.error; + log.trace('header validation error', { error: err }); + return _errorActions( + err, + websiteConfig.getErrorDocument(), + routingRules, + bucket, + reqObjectKey, + corsHeaders, + request, + log, + callback, + ); + } + // check if object to serve has website redirect header + // Note: AWS prioritizes website configuration rules over + // object key's website redirect header, so we make the + // check at the end. + if (objMD['x-amz-website-redirect-location']) { + const redirectLocation = objMD['x-amz-website-redirect-location']; + const redirectInfo = extractRedirectInfo(redirectLocation); + log.trace('redirecting to x-amz-website-redirect-location', { location: redirectLocation }); + return callback(null, false, null, corsHeaders, redirectInfo, ''); + } + // got obj metadata, authorized and headers validated, + // good to go + const responseMetaHeaders = collectResponseHeaders(objMD, corsHeaders); + + if (request.method === 'HEAD') { + pushMetric('headObject', log, { bucket: bucketName }); + monitoring.promMetrics('HEAD', bucketName, '200', 'headObject'); + return callback(null, false, null, responseMetaHeaders); + } - const dataLocator = objMD.location; - if (objMD['x-amz-server-side-encryption']) { - for (let i = 0; i < dataLocator.length; i++) { - dataLocator[i].masterKeyId = - objMD['x-amz-server-side-encryption-aws-' + - 'kms-key-id']; - dataLocator[i].algorithm = - objMD['x-amz-server-side-encryption']; - } + const dataLocator = objMD.location; + if (objMD['x-amz-server-side-encryption']) { + for (let i = 0; i < dataLocator.length; i++) { + dataLocator[i].masterKeyId = objMD['x-amz-server-side-encryption-aws-' + 'kms-key-id']; + dataLocator[i].algorithm = objMD['x-amz-server-side-encryption']; } - pushMetric('getObject', log, { - bucket: bucketName, - newByteLength: responseMetaHeaders['Content-Length'], - }); - monitoring.promMetrics('GET', bucketName, '200', - 'getObject', responseMetaHeaders['Content-Length']); - return callback(null, false, dataLocator, responseMetaHeaders); + } + pushMetric('getObject', log, { + bucket: bucketName, + newByteLength: responseMetaHeaders['Content-Length'], }); + monitoring.promMetrics('GET', bucketName, '200', 'getObject', responseMetaHeaders['Content-Length']); + return callback(null, false, dataLocator, responseMetaHeaders); + }); } return runWebsite(); diff --git a/lib/auth/in_memory/builder.js b/lib/auth/in_memory/builder.js index 37b358e6d5..068b58ceb4 100644 --- a/lib/auth/in_memory/builder.js +++ b/lib/auth/in_memory/builder.js @@ -1,5 +1,4 @@ -const serviceAccountPrefix = - require('arsenal').constants.zenkoServiceAccount; +const serviceAccountPrefix = require('arsenal').constants.zenkoServiceAccount; /** build simple authdata with only one account * @param {string} accessKey - account's accessKey @@ -9,26 +8,31 @@ const serviceAccountPrefix = * @param {string} userName - account's user name * @return {object} authdata - authdata with account's accessKey and secretKey */ -function buildAuthDataAccount(accessKey, secretKey, canonicalId, serviceName, -userName) { +function buildAuthDataAccount(accessKey, secretKey, canonicalId, serviceName, userName) { // TODO: remove specific check for clueso and generate unique // canonical id's for accounts - const finalCanonicalId = canonicalId || - (serviceName ? `${serviceAccountPrefix}/${serviceName}` : - '12349df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47qwer'); + const finalCanonicalId = + canonicalId || + (serviceName + ? `${serviceAccountPrefix}/${serviceName}` + : '12349df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47qwer'); const shortid = '123456789012'; return { - accounts: [{ - name: userName || 'CustomAccount', - email: 'customaccount1@setbyenv.com', - arn: `arn:aws:iam::${shortid}:root`, - canonicalID: finalCanonicalId, - shortid, - keys: [{ - access: accessKey, - secret: secretKey, - }], - }], + accounts: [ + { + name: userName || 'CustomAccount', + email: 'customaccount1@setbyenv.com', + arn: `arn:aws:iam::${shortid}:root`, + canonicalID: finalCanonicalId, + shortid, + keys: [ + { + access: accessKey, + secret: secretKey, + }, + ], + }, + ], }; } diff --git a/lib/auth/streamingV4/V4Transform.js b/lib/auth/streamingV4/V4Transform.js index e18d20e14f..57576733a8 100644 --- a/lib/auth/streamingV4/V4Transform.js +++ b/lib/auth/streamingV4/V4Transform.js @@ -28,8 +28,7 @@ class V4Transform extends Transform { * @param {function} errCb - callback called if an error occurs */ constructor(streamingV4Params, log, errCb) { - const { accessKey, signatureFromRequest, region, scopeDate, timestamp, - credentialScope } = streamingV4Params; + const { accessKey, signatureFromRequest, region, scopeDate, timestamp, credentialScope } = streamingV4Params; super({}); this.log = log; this.errCb = errCb; @@ -79,28 +78,24 @@ class V4Transform extends Transform { this.currentMetadata.push(remainingPlusStoredMetadata); return { completeMetadata: false }; } - let fullMetadata = remainingPlusStoredMetadata.slice(0, - lineBreakIndex); + let fullMetadata = remainingPlusStoredMetadata.slice(0, lineBreakIndex); // handle extra line break on end of data chunk if (fullMetadata.length === 0) { - const chunkWithoutLeadingLineBreak = remainingPlusStoredMetadata - .slice(2); + const chunkWithoutLeadingLineBreak = remainingPlusStoredMetadata.slice(2); // find second line break lineBreakIndex = chunkWithoutLeadingLineBreak.indexOf('\r\n'); if (lineBreakIndex < 0) { this.currentMetadata.push(chunkWithoutLeadingLineBreak); return { completeMetadata: false }; } - fullMetadata = chunkWithoutLeadingLineBreak.slice(0, - lineBreakIndex); + fullMetadata = chunkWithoutLeadingLineBreak.slice(0, lineBreakIndex); } const splitMeta = fullMetadata.toString().split(';'); this.log.trace('parsed full metadata for chunk', { splitMeta }); if (splitMeta.length !== 2) { - this.log.trace('chunk body did not contain correct ' + - 'metadata format'); + this.log.trace('chunk body did not contain correct ' + 'metadata format'); return { err: errors.InvalidArgument }; } let dataSize = splitMeta[0]; @@ -132,8 +127,7 @@ class V4Transform extends Transform { completeMetadata: true, // start slice at lineBreak plus 2 to remove line break at end of // metadata piece since length of '\r\n' is 2 - unparsedChunk: remainingPlusStoredMetadata - .slice(lineBreakIndex + 2), + unparsedChunk: remainingPlusStoredMetadata.slice(lineBreakIndex + 2), }; } @@ -146,10 +140,13 @@ class V4Transform extends Transform { */ _authenticate(dataToSend, done) { // use prior sig to construct new string to sign - const stringToSign = constructChunkStringToSign(this.timestamp, - this.credentialScope, this.lastSignature, dataToSend); - this.log.trace('constructed chunk string to sign', - { stringToSign }); + const stringToSign = constructChunkStringToSign( + this.timestamp, + this.credentialScope, + this.lastSignature, + dataToSend, + ); + this.log.trace('constructed chunk string to sign', { stringToSign }); // once used prior sig to construct string to sign, reassign // lastSignature to current signature this.lastSignature = this.currentSignature; @@ -167,15 +164,16 @@ class V4Transform extends Transform { }; return vault.authenticateV4Request(vaultParams, null, {}, err => { if (err) { - this.log.trace('err from vault on streaming v4 auth', - { error: err, paramsSentToVault: vaultParams.data }); + this.log.trace('err from vault on streaming v4 auth', { + error: err, + paramsSentToVault: vaultParams.data, + }); return done(err); } return done(); }); } - /** * This function will parse the chunk into metadata and data, * use the metadata to authenticate with vault and send the @@ -199,9 +197,7 @@ class V4Transform extends Transform { } if (this.lastPieceDone) { const slice = chunk.slice(0, 10); - this.log.trace('received chunk after end.' + - 'See first 10 bytes of chunk', - { chunk: slice.toString() }); + this.log.trace('received chunk after end.' + 'See first 10 bytes of chunk', { chunk: slice.toString() }); return callback(); } let unparsedChunk = chunk; @@ -212,11 +208,9 @@ class V4Transform extends Transform { // async function done => { if (!this.haveMetadata) { - this.log.trace('do not have metadata so calling ' + - '_parseMetadata'); + this.log.trace('do not have metadata so calling ' + '_parseMetadata'); // need to parse our metadata - const parsedMetadataResults = - this._parseMetadata(unparsedChunk); + const parsedMetadataResults = this._parseMetadata(unparsedChunk); if (parsedMetadataResults.err) { return done(parsedMetadataResults.err); } @@ -250,8 +244,7 @@ class V4Transform extends Transform { } // parse just the next data piece without \r\n at the end // (therefore, minus 2) - const nextDataPiece = - unparsedChunk.slice(0, this.seekingDataSize - 2); + const nextDataPiece = unparsedChunk.slice(0, this.seekingDataSize - 2); // add parsed data piece to other currentData pieces // so that this.currentData is the full data piece nextDataPiece.copy(this.currentData, this.dataCursor); @@ -259,8 +252,7 @@ class V4Transform extends Transform { if (err) { return done(err); } - unparsedChunk = - unparsedChunk.slice(this.seekingDataSize); + unparsedChunk = unparsedChunk.slice(this.seekingDataSize); this.push(this.currentData); this.haveMetadata = false; this.seekingDataSize = -1; @@ -283,7 +275,7 @@ class V4Transform extends Transform { } // get next chunk return callback(); - } + }, ); } } diff --git a/lib/auth/streamingV4/constructChunkStringToSign.js b/lib/auth/streamingV4/constructChunkStringToSign.js index 370501fe57..6a39806f93 100644 --- a/lib/auth/streamingV4/constructChunkStringToSign.js +++ b/lib/auth/streamingV4/constructChunkStringToSign.js @@ -13,20 +13,20 @@ const constants = require('../../../constants'); * @param {string} justDataChunk - data portion of chunk * @returns {string} stringToSign */ -function constructChunkStringToSign(timestamp, - credentialScope, lastSignature, justDataChunk) { +function constructChunkStringToSign(timestamp, credentialScope, lastSignature, justDataChunk) { let currentChunkHash; // for last chunk, there will be no data, so use emptyStringHash if (!justDataChunk) { currentChunkHash = constants.emptyStringHash; } else { currentChunkHash = crypto.createHash('sha256'); - currentChunkHash = currentChunkHash - .update(justDataChunk, 'binary').digest('hex'); + currentChunkHash = currentChunkHash.update(justDataChunk, 'binary').digest('hex'); } - return `AWS4-HMAC-SHA256-PAYLOAD\n${timestamp}\n` + + return ( + `AWS4-HMAC-SHA256-PAYLOAD\n${timestamp}\n` + `${credentialScope}\n${lastSignature}\n` + - `${constants.emptyStringHash}\n${currentChunkHash}`; + `${constants.emptyStringHash}\n${currentChunkHash}` + ); } module.exports = constructChunkStringToSign; diff --git a/lib/auth/vault.js b/lib/auth/vault.js index 3aeb06eaf9..5642e17563 100644 --- a/lib/auth/vault.js +++ b/lib/auth/vault.js @@ -49,20 +49,17 @@ function getMemBackend(config) { } switch (config.backends.auth) { -case 'mem': - implName = 'vaultMem'; - client = getMemBackend(config); - break; -case 'multiple': - implName = 'vaultChain'; - client = new ChainBackend('s3', [ - getMemBackend(config), - getVaultClient(config), - ]); - break; -default: // vault - implName = 'vault'; - client = getVaultClient(config); + case 'mem': + implName = 'vaultMem'; + client = getMemBackend(config); + break; + case 'multiple': + implName = 'vaultChain'; + client = new ChainBackend('s3', [getMemBackend(config), getVaultClient(config)]); + break; + default: // vault + implName = 'vault'; + client = getVaultClient(config); } module.exports = new Vault(client, implName); diff --git a/lib/data/wrapper.js b/lib/data/wrapper.js index cf3a216d4d..68caaf7f91 100644 --- a/lib/data/wrapper.js +++ b/lib/data/wrapper.js @@ -4,8 +4,7 @@ const { config } = require('../Config'); const kms = require('../kms/wrapper'); const metadata = require('../metadata/wrapper'); const vault = require('../auth/vault'); -const locationStorageCheck = - require('../api/apiUtils/object/locationStorageCheck'); +const locationStorageCheck = require('../api/apiUtils/object/locationStorageCheck'); const { DataWrapper, MultipleBackendGateway, parseLC } = storage.data; const { DataFileInterface } = storage.data.file; const inMemory = storage.data.inMemory.datastore.backend; @@ -28,8 +27,7 @@ if (config.backends.data === 'mem') { implName = 'file'; } else if (config.backends.data === 'multiple') { const clients = parseLC(config, vault); - client = new MultipleBackendGateway( - clients, metadata, locationStorageCheck); + client = new MultipleBackendGateway(clients, metadata, locationStorageCheck); implName = 'multipleBackends'; } else if (config.backends.data === 'cdmi') { if (!CdmiData) { @@ -45,14 +43,12 @@ if (config.backends.data === 'mem') { implName = 'cdmi'; } -const data = new DataWrapper( - client, implName, config, kms, metadata, locationStorageCheck, vault); +const data = new DataWrapper(client, implName, config, kms, metadata, locationStorageCheck, vault); config.on('location-constraints-update', () => { if (implName === 'multipleBackends') { const clients = parseLC(config, vault); - client = new MultipleBackendGateway( - clients, metadata, locationStorageCheck); + client = new MultipleBackendGateway(clients, metadata, locationStorageCheck); data.switch(client); } }); diff --git a/lib/kms/Cache.js b/lib/kms/Cache.js index c0ef71942a..43c8b8dbf2 100644 --- a/lib/kms/Cache.js +++ b/lib/kms/Cache.js @@ -41,7 +41,8 @@ class Cache { * @param {number} duration - Duration in milliseconds for cache validity. * @returns {boolean} true if the cache should be refreshed, else false. */ - shouldRefresh(duration = 1 * 60 * 60 * 1000) { // Default: 1 hour + shouldRefresh(duration = 1 * 60 * 60 * 1000) { + // Default: 1 hour if (!this.lastChecked) { return true; } @@ -49,7 +50,7 @@ class Cache { const now = Date.now(); const elapsed = now - this.lastChecked; const jitter = Math.floor(Math.random() * 15 * 60 * 1000); // Up to 15 minutes - return elapsed > (duration - jitter); + return elapsed > duration - jitter; } /** diff --git a/lib/kms/common.js b/lib/kms/common.js index 2c0cd3658d..50e87308a7 100644 --- a/lib/kms/common.js +++ b/lib/kms/common.js @@ -57,16 +57,16 @@ class Common { return newIV; } - /** - * Derive key to use in cipher - * @param {number} cryptoScheme - cryptoScheme being used - * @param {buffer} dataKey - the unencrypted key (either from the - * appliance on a get or originally generated by kms in the case of a put) - * @param {object} log - logger object - * @param {function} cb - cb from createDecipher - * @returns {undefined} - * @callback called with (err, derivedKey, derivedIV) - */ + /** + * Derive key to use in cipher + * @param {number} cryptoScheme - cryptoScheme being used + * @param {buffer} dataKey - the unencrypted key (either from the + * appliance on a get or originally generated by kms in the case of a put) + * @param {object} log - logger object + * @param {function} cb - cb from createDecipher + * @returns {undefined} + * @callback called with (err, derivedKey, derivedIV) + */ static _deriveKey(cryptoScheme, dataKey, log, cb) { if (cryptoScheme <= 1) { /* we are not storing hashed human password. @@ -79,69 +79,59 @@ class Common { */ const salt = Buffer.from('ItsTasty', 'utf8'); const iterations = 1; - return crypto.pbkdf2( - dataKey, salt, iterations, - this._keySize(), 'sha1', (err, derivedKey) => { + return crypto.pbkdf2(dataKey, salt, iterations, this._keySize(), 'sha1', (err, derivedKey) => { + if (err) { + log.error('pbkdf2 function failed on key derivation', { error: err }); + cb(errors.InternalError); + return; + } + crypto.pbkdf2(derivedKey, salt, iterations, this._IVSize(), 'sha1', (err, derivedIV) => { if (err) { - log.error('pbkdf2 function failed on key derivation', - { error: err }); - cb(errors.InternalError); - return; + log.error('pbkdf2 function failed on IV derivation', { error: err }); + return cb(errors.InternalError); } - crypto.pbkdf2( - derivedKey, salt, iterations, - this._IVSize(), 'sha1', (err, derivedIV) => { - if (err) { - log.error( - 'pbkdf2 function failed on IV derivation', - { error: err }); - return cb(errors.InternalError); - } - // derivedKey is the actual data encryption or - // decryption key used in the AES ctr cipher - return cb(null, derivedKey, derivedIV); - }); + // derivedKey is the actual data encryption or + // decryption key used in the AES ctr cipher + return cb(null, derivedKey, derivedIV); }); + }); } log.error('Unknown cryptographic scheme', { cryptoScheme }); return cb(errors.InternalError); } - /** - * createDecipher - * @param {number} cryptoScheme - cryptoScheme being used - * @param {buffer} dataKey - the unencrypted key (either from the - * appliance on a get or originally generated by kms in the case of a put) - * @param {number} offset - offset - * @param {object} log - logger object - * @param {function} cb - cb from external call - * @returns {undefined} - * @callback called with (err, decipher: ReadWritable.stream) - */ + /** + * createDecipher + * @param {number} cryptoScheme - cryptoScheme being used + * @param {buffer} dataKey - the unencrypted key (either from the + * appliance on a get or originally generated by kms in the case of a put) + * @param {number} offset - offset + * @param {object} log - logger object + * @param {function} cb - cb from external call + * @returns {undefined} + * @callback called with (err, decipher: ReadWritable.stream) + */ static createDecipher(cryptoScheme, dataKey, offset, log, cb) { - this._deriveKey( - cryptoScheme, dataKey, log, - (err, derivedKey, derivedIV) => { - if (err) { - log.debug('key derivation failed', { error: err }); - return cb(err); - } - const aesBlockSize = this._aesBlockSize(); - const blocks = Math.floor(offset / aesBlockSize); - const toSkip = offset % aesBlockSize; - const iv = this._incrementIV(derivedIV, blocks); - const cipher = crypto.createDecipheriv(this._algorithm(), - derivedKey, iv); - if (toSkip) { - /* Above, we advanced to the latest boundary not + this._deriveKey(cryptoScheme, dataKey, log, (err, derivedKey, derivedIV) => { + if (err) { + log.debug('key derivation failed', { error: err }); + return cb(err); + } + const aesBlockSize = this._aesBlockSize(); + const blocks = Math.floor(offset / aesBlockSize); + const toSkip = offset % aesBlockSize; + const iv = this._incrementIV(derivedIV, blocks); + const cipher = crypto.createDecipheriv(this._algorithm(), derivedKey, iv); + if (toSkip) { + /* Above, we advanced to the latest boundary not greater than the offset amount. Here we advance by the toSkip amount if necessary. */ - const dummyBuffer = Buffer.alloc(toSkip); - cipher.write(dummyBuffer); - cipher.read(); - } - return cb(null, cipher); - }); + const dummyBuffer = Buffer.alloc(toSkip); + cipher.write(dummyBuffer); + cipher.read(); + } + return cb(null, cipher); + }); } /** diff --git a/lib/kms/file/backend.js b/lib/kms/file/backend.js index 5f430b21ad..f7384f68b5 100644 --- a/lib/kms/file/backend.js +++ b/lib/kms/file/backend.js @@ -17,7 +17,7 @@ const backend = { * @param {function} cb - callback * @returns {undefined} * @callback called with (err, masterKeyId: string) - */ + */ createBucketKey: function createBucketKeyMem(bucket, log, cb) { process.nextTick(() => { // Using createDataKey here for purposes of createBucketKeyMem @@ -44,88 +44,69 @@ const backend = { }); }, - /** - * - * @param {number} cryptoScheme - crypto scheme version number - * @param {string} masterKeyIdOrArn - master key; for the file backend - * the master key is the actual bucket master key rather than the key to - * retrieve the actual key from a dictionary - * @param {buffer} plainTextDataKey - data key - * @param {object} log - logger object - * @param {function} cb - callback - * @returns {undefined} - * @callback called with (err, cipheredDataKey: Buffer) - */ - cipherDataKey: function cipherDataKeyMem(cryptoScheme, - masterKeyIdOrArn, - plainTextDataKey, - log, - cb) { + /** + * + * @param {number} cryptoScheme - crypto scheme version number + * @param {string} masterKeyIdOrArn - master key; for the file backend + * the master key is the actual bucket master key rather than the key to + * retrieve the actual key from a dictionary + * @param {buffer} plainTextDataKey - data key + * @param {object} log - logger object + * @param {function} cb - callback + * @returns {undefined} + * @callback called with (err, cipheredDataKey: Buffer) + */ + cipherDataKey: function cipherDataKeyMem(cryptoScheme, masterKeyIdOrArn, plainTextDataKey, log, cb) { process.nextTick(() => { const masterKeyId = getKeyIdFromArn(masterKeyIdOrArn); const masterKey = Buffer.from(masterKeyId, 'hex'); - Common.createCipher( - cryptoScheme, masterKey, 0, log, - (err, cipher) => { - if (err) { - cb(err); - return; - } - let cipheredDataKey = - cipher.update(plainTextDataKey); - // call final() to ensure that any bytes remaining in - // the output of the stream are captured - const final = cipher.final(); - if (final.length !== 0) { - cipheredDataKey = - Buffer.concat([cipheredDataKey, - final]); - } - cb(null, cipheredDataKey); - }); + Common.createCipher(cryptoScheme, masterKey, 0, log, (err, cipher) => { + if (err) { + cb(err); + return; + } + let cipheredDataKey = cipher.update(plainTextDataKey); + // call final() to ensure that any bytes remaining in + // the output of the stream are captured + const final = cipher.final(); + if (final.length !== 0) { + cipheredDataKey = Buffer.concat([cipheredDataKey, final]); + } + cb(null, cipheredDataKey); + }); }); }, - /** - * - * @param {number} cryptoScheme - crypto scheme version number - * @param {string} masterKeyIdOrArn - master key; for the file backend - * the master key is the actual bucket master key rather than the key to - * retrieve the actual key from a dictionary - * @param {buffer} cipheredDataKey - data key - * @param {object} log - logger object - * @param {function} cb - callback - * @returns {undefined} - * @callback called with (err, plainTextDataKey: Buffer) - */ - decipherDataKey: function decipherDataKeyMem(cryptoScheme, - masterKeyIdOrArn, - cipheredDataKey, - log, - cb) { + /** + * + * @param {number} cryptoScheme - crypto scheme version number + * @param {string} masterKeyIdOrArn - master key; for the file backend + * the master key is the actual bucket master key rather than the key to + * retrieve the actual key from a dictionary + * @param {buffer} cipheredDataKey - data key + * @param {object} log - logger object + * @param {function} cb - callback + * @returns {undefined} + * @callback called with (err, plainTextDataKey: Buffer) + */ + decipherDataKey: function decipherDataKeyMem(cryptoScheme, masterKeyIdOrArn, cipheredDataKey, log, cb) { process.nextTick(() => { const masterKeyId = getKeyIdFromArn(masterKeyIdOrArn); const masterKey = Buffer.from(masterKeyId, 'hex'); - Common.createDecipher( - cryptoScheme, masterKey, 0, log, - (err, decipher) => { - if (err) { - cb(err); - return; - } - let plainTextDataKey = - decipher.update(cipheredDataKey); - const final = decipher.final(); - if (final.length !== 0) { - plainTextDataKey = - Buffer.concat([plainTextDataKey, - final]); - } - cb(null, plainTextDataKey); - }); + Common.createDecipher(cryptoScheme, masterKey, 0, log, (err, decipher) => { + if (err) { + cb(err); + return; + } + let plainTextDataKey = decipher.update(cipheredDataKey); + const final = decipher.final(); + if (final.length !== 0) { + plainTextDataKey = Buffer.concat([plainTextDataKey, final]); + } + cb(null, plainTextDataKey); + }); }); }, - }; module.exports = backend; diff --git a/lib/kms/in_memory/backend.js b/lib/kms/in_memory/backend.js index 609ab17333..15e76e519c 100644 --- a/lib/kms/in_memory/backend.js +++ b/lib/kms/in_memory/backend.js @@ -15,14 +15,14 @@ const backend = { supportsDefaultKeyPerAccount: false, - /** - * - * @param {BucketInfo} bucket - bucket info - * @param {object} log - logger object - * @param {function} cb - callback - * @returns {undefined} - * @callback called with (err, masterKeyId: string) - */ + /** + * + * @param {BucketInfo} bucket - bucket info + * @param {object} log - logger object + * @param {function} cb - callback + * @returns {undefined} + * @callback called with (err, masterKeyId: string) + */ createBucketKey: function createBucketKeyMem(bucket, log, cb) { process.nextTick(() => { // Using createDataKey here for purposes of createBucketKeyMem @@ -49,82 +49,63 @@ const backend = { }); }, - /** - * - * @param {number} cryptoScheme - crypto scheme version number - * @param {string} masterKeyIdOrArn - key to retrieve master key - * @param {buffer} plainTextDataKey - data key - * @param {object} log - logger object - * @param {function} cb - callback - * @returns {undefined} - * @callback called with (err, cipheredDataKey: Buffer) - */ - cipherDataKey: function cipherDataKeyMem(cryptoScheme, - masterKeyIdOrArn, - plainTextDataKey, - log, - cb) { + /** + * + * @param {number} cryptoScheme - crypto scheme version number + * @param {string} masterKeyIdOrArn - key to retrieve master key + * @param {buffer} plainTextDataKey - data key + * @param {object} log - logger object + * @param {function} cb - callback + * @returns {undefined} + * @callback called with (err, cipheredDataKey: Buffer) + */ + cipherDataKey: function cipherDataKeyMem(cryptoScheme, masterKeyIdOrArn, plainTextDataKey, log, cb) { process.nextTick(() => { const masterKeyId = getKeyIdFromArn(masterKeyIdOrArn); - Common.createCipher( - cryptoScheme, kms[masterKeyId], 0, log, - (err, cipher) => { - if (err) { - cb(err); - return; - } - let cipheredDataKey = - cipher.update(plainTextDataKey); - // call final() to ensure that any bytes remaining in - // the output of the stream are captured - const final = cipher.final(); - if (final.length !== 0) { - cipheredDataKey = - Buffer.concat([cipheredDataKey, - final]); - } - cb(null, cipheredDataKey); - }); + Common.createCipher(cryptoScheme, kms[masterKeyId], 0, log, (err, cipher) => { + if (err) { + cb(err); + return; + } + let cipheredDataKey = cipher.update(plainTextDataKey); + // call final() to ensure that any bytes remaining in + // the output of the stream are captured + const final = cipher.final(); + if (final.length !== 0) { + cipheredDataKey = Buffer.concat([cipheredDataKey, final]); + } + cb(null, cipheredDataKey); + }); }); }, - /** - * - * @param {number} cryptoScheme - crypto scheme version number - * @param {string} masterKeyIdOrArn - key to retrieve master key - * @param {buffer} cipheredDataKey - data key - * @param {object} log - logger object - * @param {function} cb - callback - * @returns {undefined} - * @callback called with (err, plainTextDataKey: Buffer) - */ - decipherDataKey: function decipherDataKeyMem(cryptoScheme, - masterKeyIdOrArn, - cipheredDataKey, - log, - cb) { + /** + * + * @param {number} cryptoScheme - crypto scheme version number + * @param {string} masterKeyIdOrArn - key to retrieve master key + * @param {buffer} cipheredDataKey - data key + * @param {object} log - logger object + * @param {function} cb - callback + * @returns {undefined} + * @callback called with (err, plainTextDataKey: Buffer) + */ + decipherDataKey: function decipherDataKeyMem(cryptoScheme, masterKeyIdOrArn, cipheredDataKey, log, cb) { process.nextTick(() => { const masterKeyId = getKeyIdFromArn(masterKeyIdOrArn); - Common.createDecipher( - cryptoScheme, kms[masterKeyId], 0, log, - (err, decipher) => { - if (err) { - cb(err); - return; - } - let plainTextDataKey = - decipher.update(cipheredDataKey); - const final = decipher.final(); - if (final.length !== 0) { - plainTextDataKey = - Buffer.concat([plainTextDataKey, - final]); - } - cb(null, plainTextDataKey); - }); + Common.createDecipher(cryptoScheme, kms[masterKeyId], 0, log, (err, decipher) => { + if (err) { + cb(err); + return; + } + let plainTextDataKey = decipher.update(cipheredDataKey); + const final = decipher.final(); + if (final.length !== 0) { + plainTextDataKey = Buffer.concat([plainTextDataKey, final]); + } + cb(null, plainTextDataKey); + }); }); }, - }; module.exports = { diff --git a/lib/kms/utilities.js b/lib/kms/utilities.js index 5cdfa1f3bc..4c290528e0 100644 --- a/lib/kms/utilities.js +++ b/lib/kms/utilities.js @@ -5,13 +5,7 @@ const http = require('http'); const https = require('https'); const logger = require('../utilities/logger'); -function _createEncryptedBucket(host, - port, - bucketName, - accessKey, - secretKey, - verbose, ssl, - locationConstraint) { +function _createEncryptedBucket(host, port, bucketName, accessKey, secretKey, verbose, ssl, locationConstraint) { const options = { host, port, @@ -55,10 +49,11 @@ function _createEncryptedBucket(host, logger.info('request headers', { headers: request.getHeaders() }); } if (locationConstraint) { - const createBucketConfiguration = '' + - `${locationConstraint}` + - ''; + const createBucketConfiguration = + '' + + `${locationConstraint}` + + ''; request.write(createBucketConfiguration); } request.end(); @@ -81,20 +76,17 @@ function createEncryptedBucket() { .option('-p, --port ', 'Port of the server') .option('-s, --ssl', 'Enable ssl') .option('-v, --verbose') - .option('-l, --location-constraint ', - 'location Constraint') + .option('-l, --location-constraint ', 'location Constraint') .parse(process.argv); - const { host, port, accessKey, secretKey, bucket, verbose, ssl, - locationConstraint } = commander; + const { host, port, accessKey, secretKey, bucket, verbose, ssl, locationConstraint } = commander; if (!host || !port || !accessKey || !secretKey || !bucket) { logger.error('missing parameter'); commander.outputHelp(); process.exit(1); } - _createEncryptedBucket(host, port, bucket, accessKey, secretKey, verbose, - ssl, locationConstraint); + _createEncryptedBucket(host, port, bucket, accessKey, secretKey, verbose, ssl, locationConstraint); } module.exports = { diff --git a/lib/kms/wrapper.js b/lib/kms/wrapper.js index 772f6d2204..c2db3f2a06 100644 --- a/lib/kms/wrapper.js +++ b/lib/kms/wrapper.js @@ -30,9 +30,7 @@ function getScalityKms() { scalityKMS = new ScalityKMS(config.kms); scalityKMSImpl = 'scalityKms'; } catch (error) { - logger.warn('scality kms unavailable. ' + - 'Using file kms backend unless mem specified.', - { error }); + logger.warn('scality kms unavailable. ' + 'Using file kms backend unless mem specified.', { error }); scalityKMS = file; scalityKMSImpl = 'fileKms'; } @@ -97,12 +95,10 @@ if (config.sseMigration) { previousBackend = makeBackend( config.sseMigration.previousKeyType, config.sseMigration.previousKeyProtocol, - config.sseMigration.previousKeyProvider + config.sseMigration.previousKeyProvider, ); availableBackends.push(previousBackend); - previousIdentifier = `${previousBackend.type - }:${previousBackend.protocol - }:${previousBackend.provider}`; + previousIdentifier = `${previousBackend.type}:${previousBackend.protocol}:${previousBackend.provider}`; // Pre instantiate previous backend as for now only internal backend (file) is supported // for future multiple external backend we should consider keeping open connection to @@ -152,10 +148,10 @@ function getClientForKey(key, log) { // Only pre instantiated previous KMS from sseMigration is supported now // Here we could instantiate other provider on the fly to manage multi providers - log.error('KMS key doesn\'t match any KMS instance', { key, detail, availableBackends }); - return { error: new errors.InvalidArgument - // eslint-disable-next-line new-cap - .customizeDescription(`KMS unknown provider for key ${key}`), + log.error("KMS key doesn't match any KMS instance", { key, detail, availableBackends }); + return { + error: new // eslint-disable-next-line new-cap + errors.InvalidArgument.customizeDescription(`KMS unknown provider for key ${key}`), }; } @@ -170,20 +166,20 @@ class KMS { return client.backend.arnPrefix; } - /** - * Create a new bucket encryption key. - * - * This function is responsible for creating an encryption key for a bucket. - * If the client supports using a default master encryption key per account - * and one is configured, the key is managed at the account level by Vault. - * Otherwise, a bucket-level encryption key is created for legacy support. - * - * @param {BucketInfo} bucket - bucket info - * @param {object} log - logger object - * @param {function} cb - callback - * @returns {undefined} - * @callback called with (err, { masterKeyId: string, masterKeyArn: string, isAccountEncryptionEnabled: boolean }) - */ + /** + * Create a new bucket encryption key. + * + * This function is responsible for creating an encryption key for a bucket. + * If the client supports using a default master encryption key per account + * and one is configured, the key is managed at the account level by Vault. + * Otherwise, a bucket-level encryption key is created for legacy support. + * + * @param {BucketInfo} bucket - bucket info + * @param {object} log - logger object + * @param {function} cb - callback + * @returns {undefined} + * @callback called with (err, { masterKeyId: string, masterKeyArn: string, isAccountEncryptionEnabled: boolean }) + */ static createBucketKey(bucket, log, cb) { // always use current client for create log.debug('creating a new bucket key'); @@ -194,14 +190,19 @@ class KMS { if (client.supportsDefaultKeyPerAccount && config.defaultEncryptionKeyPerAccount) { return vault.getOrCreateEncryptionKeyId(bucket.getOwner(), log, (err, data) => { if (err) { - log.debug('error retrieving or creating the default encryption key at the account level from vault', - { implName, error: err }); + log.debug( + 'error retrieving or creating the default encryption key at the account level from vault', + { implName, error: err }, + ); return cb(err); } const { encryptionKeyId, action } = data; - log.trace('default encryption key retrieved or created at the account level from vault', - { implName, encryptionKeyId, action }); + log.trace('default encryption key retrieved or created at the account level from vault', { + implName, + encryptionKeyId, + action, + }); return cb(null, { // vault only return arn masterKeyId: encryptionKeyId, @@ -221,15 +222,15 @@ class KMS { }); } - /** - * - * @param {BucketInfo} bucket - bucket info - * @param {object} sseConfig - SSE configuration - * @param {object} log - logger object - * @param {function} cb - callback - * @returns {undefined} - * @callback called with (err, serverSideEncryptionInfo: object) - */ + /** + * + * @param {BucketInfo} bucket - bucket info + * @param {object} sseConfig - SSE configuration + * @param {object} log - logger object + * @param {function} cb - callback + * @returns {undefined} + * @callback called with (err, serverSideEncryptionInfo: object) + */ static bucketLevelEncryption(bucket, sseConfig, log, cb) { /* The purpose of bucket level encryption is so that the client does not @@ -258,8 +259,7 @@ class KMS { } serverSideEncryptionInfo.configuredMasterKeyId = configuredMasterKeyId; } else { - serverSideEncryptionInfo.configuredMasterKeyId = - `${client.backend.arnPrefix}${configuredMasterKeyId}`; + serverSideEncryptionInfo.configuredMasterKeyId = `${client.backend.arnPrefix}${configuredMasterKeyId}`; } return process.nextTick(() => cb(null, serverSideEncryptionInfo)); @@ -280,9 +280,9 @@ class KMS { return cb(null, serverSideEncryptionInfo); }); } - /* - * no encryption - */ + /* + * no encryption + */ return cb(null, null); } @@ -311,26 +311,25 @@ class KMS { }); } - /** - * createCipherBundle - * @param {object} serverSideEncryptionInfo - info for encryption - * @param {number} serverSideEncryptionInfo.cryptoScheme - - * cryptoScheme used - * @param {string} serverSideEncryptionInfo.algorithm - - * algorithm to use - * @param {string} serverSideEncryptionInfo.masterKeyId - - * key to get master key - * @param {boolean} serverSideEncryptionInfo.mandatory - - * true for mandatory encryption - * @param {object} log - logger object - * @param {function} cb - cb from external call - * @param {object} [opts] - additional options - * @param {boolean} [opts.previousOk] - allow usage of previous KMS (for ongoing MPU not migrated) - * @returns {undefined} - * @callback called with (err, cipherBundle) - */ - static createCipherBundle(serverSideEncryptionInfo, - log, cb, opts) { + /** + * createCipherBundle + * @param {object} serverSideEncryptionInfo - info for encryption + * @param {number} serverSideEncryptionInfo.cryptoScheme - + * cryptoScheme used + * @param {string} serverSideEncryptionInfo.algorithm - + * algorithm to use + * @param {string} serverSideEncryptionInfo.masterKeyId - + * key to get master key + * @param {boolean} serverSideEncryptionInfo.mandatory - + * true for mandatory encryption + * @param {object} log - logger object + * @param {function} cb - cb from external call + * @param {object} [opts] - additional options + * @param {boolean} [opts.previousOk] - allow usage of previous KMS (for ongoing MPU not migrated) + * @returns {undefined} + * @callback called with (err, cipherBundle) + */ + static createCipherBundle(serverSideEncryptionInfo, log, cb, opts) { const { algorithm, configuredMasterKeyId, masterKeyId: bucketMasterKeyId } = serverSideEncryptionInfo; let masterKeyId = bucketMasterKeyId; @@ -345,14 +344,18 @@ class KMS { if (error) { return cb(error); } - if (previousIdentifier - && clientIdentifier === previousIdentifier - && clientIdentifier !== currentIdentifier - && (opts && !opts.previousOk) + if ( + previousIdentifier && + clientIdentifier === previousIdentifier && + clientIdentifier !== currentIdentifier && + opts && + !opts.previousOk ) { - return cb(errors.InvalidArgument - .customizeDescription( - 'KMS cannot use previous provider to encrypt new objects if a new provider is configured')); + return cb( + errors.InvalidArgument.customizeDescription( + 'KMS cannot use previous provider to encrypt new objects if a new provider is configured', + ), + ); } const cipherBundle = { @@ -363,9 +366,10 @@ class KMS { cipher: null, }; - return async.waterfall([ - function generateDataKey(next) { - /* There are 2 ways of generating a datakey : + return async.waterfall( + [ + function generateDataKey(next) { + /* There are 2 ways of generating a datakey : - using the generateDataKey of the KMS backend if it exists (currently only implemented for the AWS KMS backend). This is the preferred solution since a dedicated KMS should offer a better @@ -374,92 +378,96 @@ class KMS { encrypt the datakey. This method is used when the KMS backend doesn't provide the generateDataKey method. */ - let res; - if (client.generateDataKey) { - log.debug('creating a data key using the KMS'); - res = client.generateDataKey(cipherBundle.cryptoScheme, - key, - log, (err, plainTextDataKey, cipheredDataKey) => { - if (err) { - log.debug('error generating a new data key from KMS', - { implName, error: err }); - return next(err); - } - log.trace('data key generated by the kms'); - return next(null, plainTextDataKey, cipheredDataKey); - }); - } else { - log.debug('creating a new data key'); - const plainTextDataKey = Common.createDataKey(); - - log.debug('ciphering the data key'); - res = client.cipherDataKey(cipherBundle.cryptoScheme, - key, - plainTextDataKey, log, (err, cipheredDataKey) => { - if (err) { - log.debug('error encrypting the data key using KMS', - { implName, error: err }); - return next(err); - } - log.trace('data key ciphered by the kms'); - return next(null, plainTextDataKey, cipheredDataKey); - }); - } - return res; - }, - function createCipher(plainTextDataKey, cipheredDataKey, next) { - log.debug('creating a cipher'); - cipherBundle.cipheredDataKey = - cipheredDataKey.toString('base64'); - return Common.createCipher(cipherBundle.cryptoScheme, - plainTextDataKey, 0, log, (err, cipher) => { + let res; + if (client.generateDataKey) { + log.debug('creating a data key using the KMS'); + res = client.generateDataKey( + cipherBundle.cryptoScheme, + key, + log, + (err, plainTextDataKey, cipheredDataKey) => { + if (err) { + log.debug('error generating a new data key from KMS', { implName, error: err }); + return next(err); + } + log.trace('data key generated by the kms'); + return next(null, plainTextDataKey, cipheredDataKey); + }, + ); + } else { + log.debug('creating a new data key'); + const plainTextDataKey = Common.createDataKey(); + + log.debug('ciphering the data key'); + res = client.cipherDataKey( + cipherBundle.cryptoScheme, + key, + plainTextDataKey, + log, + (err, cipheredDataKey) => { + if (err) { + log.debug('error encrypting the data key using KMS', { implName, error: err }); + return next(err); + } + log.trace('data key ciphered by the kms'); + return next(null, plainTextDataKey, cipheredDataKey); + }, + ); + } + return res; + }, + function createCipher(plainTextDataKey, cipheredDataKey, next) { + log.debug('creating a cipher'); + cipherBundle.cipheredDataKey = cipheredDataKey.toString('base64'); + return Common.createCipher(cipherBundle.cryptoScheme, plainTextDataKey, 0, log, (err, cipher) => { plainTextDataKey.fill(0); if (err) { - log.debug('error from kms', - { implName, error: err }); + log.debug('error from kms', { implName, error: err }); return next(err); } log.trace('cipher created by the kms'); return next(null, cipher); }); + }, + function finishCipherBundle(cipher, next) { + cipherBundle.cipher = cipher; + return next(null, cipherBundle); + }, + ], + (err, cipherBundle) => { + if (err) { + log.error('error processing cipher bundle', { implName, error: err }); + } + return cb(err, cipherBundle); }, - function finishCipherBundle(cipher, next) { - cipherBundle.cipher = cipher; - return next(null, cipherBundle); - }, - ], (err, cipherBundle) => { - if (err) { - log.error('error processing cipher bundle', - { implName, error: err }); - } - return cb(err, cipherBundle); - }); + ); } - /** - * createDecipherBundle - * @param {object} serverSideEncryptionInfo - info for decryption - * @param {number} serverSideEncryptionInfo.cryptoScheme - - * cryptoScheme used - * @param {string} serverSideEncryptionInfo.algorithm - - * algorithm to use - * @param {string} serverSideEncryptionInfo.masterKeyId - - * key to get master key - * @param {boolean} serverSideEncryptionInfo.mandatory - - * true for mandatory encryption - * @param {buffer} serverSideEncryptionInfo.cipheredDataKey - - * ciphered data key - * @param {number} offset - offset for decryption - * @param {object} log - logger object - * @param {function} cb - cb from external call - * @returns {undefined} - * @callback called with (err, decipherBundle) - */ - static createDecipherBundle(serverSideEncryptionInfo, offset, - log, cb) { - if (!serverSideEncryptionInfo.masterKeyId || + /** + * createDecipherBundle + * @param {object} serverSideEncryptionInfo - info for decryption + * @param {number} serverSideEncryptionInfo.cryptoScheme - + * cryptoScheme used + * @param {string} serverSideEncryptionInfo.algorithm - + * algorithm to use + * @param {string} serverSideEncryptionInfo.masterKeyId - + * key to get master key + * @param {boolean} serverSideEncryptionInfo.mandatory - + * true for mandatory encryption + * @param {buffer} serverSideEncryptionInfo.cipheredDataKey - + * ciphered data key + * @param {number} offset - offset for decryption + * @param {object} log - logger object + * @param {function} cb - cb from external call + * @returns {undefined} + * @callback called with (err, decipherBundle) + */ + static createDecipherBundle(serverSideEncryptionInfo, offset, log, cb) { + if ( + !serverSideEncryptionInfo.masterKeyId || !serverSideEncryptionInfo.cipheredDataKey || - !serverSideEncryptionInfo.cryptoScheme) { + !serverSideEncryptionInfo.cryptoScheme + ) { log.error('Invalid cryptographic information', { implName }); return cb(errors.InternalError); } @@ -469,55 +477,61 @@ class KMS { }; // shadowing global client for key - implName already used can't be shadowed here - const { error, client, implName: _impl, key } = getClientForKey( - serverSideEncryptionInfo.masterKeyId, log); + const { error, client, implName: _impl, key } = getClientForKey(serverSideEncryptionInfo.masterKeyId, log); if (error) { return cb(error); } - return async.waterfall([ - function decipherDataKey(next) { - return client.decipherDataKey( - decipherBundle.cryptoScheme, - key, - serverSideEncryptionInfo.cipheredDataKey, - log, (err, plainTextDataKey) => { - log.debug('deciphering a data key'); - if (err) { - log.debug('error from kms', - { implName: _impl, error: err }); - return next(err); - } - log.trace('data key deciphered by the kms'); - return next(null, plainTextDataKey); - }); - }, - function createDecipher(plainTextDataKey, next) { - log.debug('creating a decipher'); - return Common.createDecipher(decipherBundle.cryptoScheme, - plainTextDataKey, offset, log, (err, decipher) => { - plainTextDataKey.fill(0); - if (err) { - log.debug('error from kms', - { implName: _impl, error: err }); - return next(err); - } - log.trace('decipher created by the kms'); - return next(null, decipher); - }); - }, - function finishDecipherBundle(decipher, next) { - decipherBundle.decipher = decipher; - return next(null, decipherBundle); + return async.waterfall( + [ + function decipherDataKey(next) { + return client.decipherDataKey( + decipherBundle.cryptoScheme, + key, + serverSideEncryptionInfo.cipheredDataKey, + log, + (err, plainTextDataKey) => { + log.debug('deciphering a data key'); + if (err) { + log.debug('error from kms', { implName: _impl, error: err }); + return next(err); + } + log.trace('data key deciphered by the kms'); + return next(null, plainTextDataKey); + }, + ); + }, + function createDecipher(plainTextDataKey, next) { + log.debug('creating a decipher'); + return Common.createDecipher( + decipherBundle.cryptoScheme, + plainTextDataKey, + offset, + log, + (err, decipher) => { + plainTextDataKey.fill(0); + if (err) { + log.debug('error from kms', { implName: _impl, error: err }); + return next(err); + } + log.trace('decipher created by the kms'); + return next(null, decipher); + }, + ); + }, + function finishDecipherBundle(decipher, next) { + decipherBundle.decipher = decipher; + return next(null, decipherBundle); + }, + ], + (err, decipherBundle) => { + if (err) { + log.error('error processing decipher bundle', { implName: _impl, error: err }); + return cb(err); + } + return cb(err, decipherBundle); }, - ], (err, decipherBundle) => { - if (err) { - log.error('error processing decipher bundle', - { implName: _impl, error: err }); - return cb(err); - } - return cb(err, decipherBundle); - }); + ); } static checkHealth(log, cb) { diff --git a/lib/management/agentClient.js b/lib/management/agentClient.js index f88b199565..e10490766f 100644 --- a/lib/management/agentClient.js +++ b/lib/management/agentClient.js @@ -6,7 +6,6 @@ const _config = require('../Config').config; const { patchConfiguration } = require('./configuration'); const { reshapeExceptionError } = arsenal.errorUtils; - const managementAgentMessageType = { /** Message that contains the loaded overlay */ NEW_OVERLAY: 1, @@ -14,7 +13,6 @@ const managementAgentMessageType = { const CONNECTION_RETRY_TIMEOUT_MS = 5000; - function initManagementClient() { const { host, port } = _config.managementAgent; @@ -62,22 +60,22 @@ function initManagementClient() { } switch (msg.messageType) { - case managementAgentMessageType.NEW_OVERLAY: - patchConfiguration(msg.payload, log, err => { - if (err) { - log.error('failed to patch overlay', { - error: reshapeExceptionError(err), - method, - }); - } - }); - return; - default: - log.error('new overlay message with unmanaged message type', { - method, - type: msg.messageType, - }); - return; + case managementAgentMessageType.NEW_OVERLAY: + patchConfiguration(msg.payload, log, err => { + if (err) { + log.error('failed to patch overlay', { + error: reshapeExceptionError(err), + method, + }); + } + }); + return; + default: + log.error('new overlay message with unmanaged message type', { + method, + type: msg.messageType, + }); + return; } }); } @@ -86,7 +84,6 @@ function isManagementAgentUsed() { return process.env.MANAGEMENT_USE_AGENT === '1'; } - module.exports = { managementAgentMessageType, initManagementClient, diff --git a/lib/management/configuration.js b/lib/management/configuration.js index bd3cc08bea..de6db45995 100644 --- a/lib/management/configuration.js +++ b/lib/management/configuration.js @@ -19,9 +19,10 @@ function overlayHasVersion(overlay) { } function remoteOverlayIsNewer(cachedOverlay, remoteOverlay) { - return (overlayHasVersion(remoteOverlay) && - (!overlayHasVersion(cachedOverlay) || - remoteOverlay.version > cachedOverlay.version)); + return ( + overlayHasVersion(remoteOverlay) && + (!overlayHasVersion(cachedOverlay) || remoteOverlay.version > cachedOverlay.version) + ); } /** @@ -41,10 +42,8 @@ function patchConfiguration(newConf, log, cb) { return process.nextTick(cb, null, newConf); } - if (_config.overlayVersion !== undefined && - newConf.version <= _config.overlayVersion) { - log.debug('configuration version already applied', - { configurationVersion: newConf.version }); + if (_config.overlayVersion !== undefined && newConf.version <= _config.overlayVersion) { + log.debug('configuration version already applied', { configurationVersion: newConf.version }); return process.nextTick(cb, null, newConf); } return getStoredCredentials(log, (err, creds) => { @@ -62,8 +61,12 @@ function patchConfiguration(newConf, log, cb) { serviceName = u.accountType.split('-')[1]; } const newAccount = buildAuthDataAccount( - u.accessKey, secretKey, u.canonicalId, serviceName, - u.userName); + u.accessKey, + secretKey, + u.canonicalId, + serviceName, + u.userName, + ); accounts.push(newAccount.accounts[0]); } }); @@ -86,29 +89,30 @@ function patchConfiguration(newConf, log, cb) { _config.setLocationConstraints(locations); } catch (error) { const exceptionError = reshapeExceptionError(error); - log.error('could not apply configuration version location ' + - 'constraints', { error: exceptionError, - method: 'getStoredCredentials' }); + log.error('could not apply configuration version location ' + 'constraints', { + error: exceptionError, + method: 'getStoredCredentials', + }); return cb(exceptionError); } try { const locationsWithReplicationBackend = Object.keys(locations) - // NOTE: In Orbit, we don't need to have Scality location in our - // replication endpoind config, since we do not replicate to - // any Scality Instance yet. - .filter(key => replicationBackends - [locations[key].type]) - .reduce((obj, key) => { - /* eslint no-param-reassign:0 */ - obj[key] = locations[key]; - return obj; - }, {}); - _config.setReplicationEndpoints( - locationsWithReplicationBackend); + // NOTE: In Orbit, we don't need to have Scality location in our + // replication endpoind config, since we do not replicate to + // any Scality Instance yet. + .filter(key => replicationBackends[locations[key].type]) + .reduce((obj, key) => { + /* eslint no-param-reassign:0 */ + obj[key] = locations[key]; + return obj; + }, {}); + _config.setReplicationEndpoints(locationsWithReplicationBackend); } catch (error) { const exceptionError = reshapeExceptionError(error); - log.error('could not apply replication endpoints', - { error: exceptionError, method: 'getStoredCredentials' }); + log.error('could not apply replication endpoints', { + error: exceptionError, + method: 'getStoredCredentials', + }); return cb(exceptionError); } } @@ -118,18 +122,15 @@ function patchConfiguration(newConf, log, cb) { _config.setPublicInstanceId(newConf.instanceId); if (newConf.browserAccess) { - if (Boolean(_config.browserAccessEnabled) !== - Boolean(newConf.browserAccess.enabled)) { - _config.browserAccessEnabled = - Boolean(newConf.browserAccess.enabled); + if (Boolean(_config.browserAccessEnabled) !== Boolean(newConf.browserAccess.enabled)) { + _config.browserAccessEnabled = Boolean(newConf.browserAccess.enabled); _config.emit('browser-access-enabled-change'); } } _config.overlayVersion = newConf.version; - log.info('applied configuration version', - { configurationVersion: _config.overlayVersion }); + log.info('applied configuration version', { configurationVersion: _config.overlayVersion }); return cb(null, newConf); }); @@ -149,28 +150,33 @@ function patchConfiguration(newConf, log, cb) { function saveConfigurationVersion(cachedOverlay, remoteOverlay, log, cb) { if (remoteOverlayIsNewer(cachedOverlay, remoteOverlay)) { const objName = `configuration/overlay/${remoteOverlay.version}`; - metadata.putObjectMD(managementDatabaseName, objName, remoteOverlay, - {}, log, error => { - if (error) { - const exceptionError = reshapeExceptionError(error); - log.error('could not save configuration', - { error: exceptionError, - method: 'saveConfigurationVersion', - configurationVersion: remoteOverlay.version }); - cb(exceptionError); - return; - } - metadata.putObjectMD(managementDatabaseName, - latestOverlayVersionKey, remoteOverlay.version, {}, log, - error => { - if (error) { - log.error('could not save configuration version', { - configurationVersion: remoteOverlay.version, - }); - } - cb(error, remoteOverlay); - }); - }); + metadata.putObjectMD(managementDatabaseName, objName, remoteOverlay, {}, log, error => { + if (error) { + const exceptionError = reshapeExceptionError(error); + log.error('could not save configuration', { + error: exceptionError, + method: 'saveConfigurationVersion', + configurationVersion: remoteOverlay.version, + }); + cb(exceptionError); + return; + } + metadata.putObjectMD( + managementDatabaseName, + latestOverlayVersionKey, + remoteOverlay.version, + {}, + log, + error => { + if (error) { + log.error('could not save configuration version', { + configurationVersion: remoteOverlay.version, + }); + } + cb(error, remoteOverlay); + }, + ); + }); } else { log.debug('no remote configuration to cache yet'); process.nextTick(cb, null, remoteOverlay); @@ -187,25 +193,29 @@ function saveConfigurationVersion(cachedOverlay, remoteOverlay, log, cb) { * @returns {undefined} */ function loadCachedOverlay(log, callback) { - return metadata.getObjectMD(managementDatabaseName, - latestOverlayVersionKey, {}, log, (err, version) => { - if (err) { - if (err.is.NoSuchKey) { - return process.nextTick(callback, null, {}); - } - return callback(err); + return metadata.getObjectMD(managementDatabaseName, latestOverlayVersionKey, {}, log, (err, version) => { + if (err) { + if (err.is.NoSuchKey) { + return process.nextTick(callback, null, {}); } - return metadata.getObjectMD(managementDatabaseName, - `configuration/overlay/${version}`, {}, log, (err, conf) => { - if (err) { - if (err.is.NoSuchKey) { - return process.nextTick(callback, null, {}); - } - return callback(err); + return callback(err); + } + return metadata.getObjectMD( + managementDatabaseName, + `configuration/overlay/${version}`, + {}, + log, + (err, conf) => { + if (err) { + if (err.is.NoSuchKey) { + return process.nextTick(callback, null, {}); } - return callback(null, conf); - }); - }); + return callback(err); + } + return callback(null, conf); + }, + ); + }); } function applyAndSaveOverlay(overlay, log) { diff --git a/lib/management/credentials.js b/lib/management/credentials.js index 814a731dc9..afee99d4f4 100644 --- a/lib/management/credentials.js +++ b/lib/management/credentials.js @@ -21,8 +21,7 @@ const { reshapeExceptionError } = arsenal.errorUtils; * @returns {undefined} */ function getStoredCredentials(log, callback) { - metadata.getObjectMD(managementDatabaseName, tokenConfigurationKey, {}, - log, callback); + metadata.getObjectMD(managementDatabaseName, tokenConfigurationKey, {}, log, callback); } function issueCredentials(managementEndpoint, instanceId, log, callback) { @@ -36,8 +35,10 @@ function issueCredentials(managementEndpoint, instanceId, log, callback) { publicKey, }; - request.post(`${managementEndpoint}/${instanceId}/register`, - { body: postData, json: true }, (error, response, body) => { + request.post( + `${managementEndpoint}/${instanceId}/register`, + { body: postData, json: true }, + (error, response, body) => { if (error) { return callback(error); } @@ -51,11 +52,11 @@ function issueCredentials(managementEndpoint, instanceId, log, callback) { body.privateKey = privateKey; /* eslint-enable no-param-reassign */ return callback(null, body); - }); + }, + ); } -function confirmInstanceCredentials( - managementEndpoint, instanceId, creds, log, callback) { +function confirmInstanceCredentials(managementEndpoint, instanceId, creds, log, callback) { const postData = { serial: creds.serial || 0, publicKey: creds.publicKey, @@ -68,16 +69,15 @@ function confirmInstanceCredentials( body: postData, }; - request.post(`${managementEndpoint}/${instanceId}/confirm`, - opts, (error, response) => { - if (error) { - return callback(error); - } - if (response.statusCode === 200) { - return callback(null, instanceId, creds.token); - } - return callback(arsenal.errors.InternalError); - }); + request.post(`${managementEndpoint}/${instanceId}/confirm`, opts, (error, response) => { + if (error) { + return callback(error); + } + if (response.statusCode === 200) { + return callback(null, instanceId, creds.token); + } + return callback(arsenal.errors.InternalError); + }); } /** @@ -95,35 +95,37 @@ function confirmInstanceCredentials( * * @returns {undefined} */ -function initManagementCredentials( - managementEndpoint, instanceId, log, callback) { +function initManagementCredentials(managementEndpoint, instanceId, log, callback) { getStoredCredentials(log, (error, value) => { if (error) { if (error.is.NoSuchKey) { - return issueCredentials(managementEndpoint, instanceId, log, - (error, value) => { + return issueCredentials(managementEndpoint, instanceId, log, (error, value) => { if (error) { - log.error('could not issue token', - { error: reshapeExceptionError(error), - method: 'initManagementCredentials' }); + log.error('could not issue token', { + error: reshapeExceptionError(error), + method: 'initManagementCredentials', + }); return callback(error); } log.debug('saving token'); - return metadata.putObjectMD(managementDatabaseName, - tokenConfigurationKey, value, {}, log, error => { + return metadata.putObjectMD( + managementDatabaseName, + tokenConfigurationKey, + value, + {}, + log, + error => { if (error) { - log.error('could not save token', - { error: reshapeExceptionError(error), - method: 'initManagementCredentials', - }); + log.error('could not save token', { + error: reshapeExceptionError(error), + method: 'initManagementCredentials', + }); return callback(error); } - log.info('saved token locally, ' + - 'confirming instance'); - return confirmInstanceCredentials( - managementEndpoint, instanceId, value, log, - callback); - }); + log.info('saved token locally, ' + 'confirming instance'); + return confirmInstanceCredentials(managementEndpoint, instanceId, value, log, callback); + }, + ); }); } log.debug('could not get token', { error }); diff --git a/lib/management/index.js b/lib/management/index.js index d98f3c46bc..2b8b7d6755 100644 --- a/lib/management/index.js +++ b/lib/management/index.js @@ -4,11 +4,7 @@ const async = require('async'); const metadata = require('../metadata/wrapper'); const logger = require('../utilities/logger'); -const { - loadCachedOverlay, - managementDatabaseName, - patchConfiguration, -} = require('./configuration'); +const { loadCachedOverlay, managementDatabaseName, patchConfiguration } = require('./configuration'); const { initManagementCredentials } = require('./credentials'); const { startWSManagementClient } = require('./push'); const { startPollingManagementClient } = require('./poll'); @@ -17,20 +13,20 @@ const { isManagementAgentUsed } = require('./agentClient'); const initRemoteManagementRetryDelay = 10000; -const managementEndpointRoot = - process.env.MANAGEMENT_ENDPOINT || - 'https://api.zenko.io'; +const managementEndpointRoot = process.env.MANAGEMENT_ENDPOINT || 'https://api.zenko.io'; const managementEndpoint = `${managementEndpointRoot}/api/v1/instance`; -const pushEndpointRoot = - process.env.PUSH_ENDPOINT || - 'https://push.api.zenko.io'; +const pushEndpointRoot = process.env.PUSH_ENDPOINT || 'https://push.api.zenko.io'; const pushEndpoint = `${pushEndpointRoot}/api/v1/instance`; function initManagementDatabase(log, callback) { // XXX choose proper owner names - const md = new arsenal.models.BucketInfo(managementDatabaseName, 'owner', - 'owner display name', new Date().toJSON()); + const md = new arsenal.models.BucketInfo( + managementDatabaseName, + 'owner', + 'owner display name', + new Date().toJSON(), + ); metadata.createBucket(managementDatabaseName, md, log, error => { if (error) { @@ -38,9 +34,10 @@ function initManagementDatabase(log, callback) { log.info('created management database'); return callback(); } - log.error('could not initialize management database', - { error: reshapeExceptionError(error), - method: 'initManagementDatabase' }); + log.error('could not initialize management database', { + error: reshapeExceptionError(error), + method: 'initManagementDatabase', + }); return callback(error); } log.info('initialized management database'); @@ -75,61 +72,63 @@ function startManagementListeners(instanceId, token) { * @returns {undefined} */ function initManagement(log, callback) { - if ((process.env.REMOTE_MANAGEMENT_DISABLE && - process.env.REMOTE_MANAGEMENT_DISABLE !== '0') - || process.env.S3BACKEND === 'mem') { + if ( + (process.env.REMOTE_MANAGEMENT_DISABLE && process.env.REMOTE_MANAGEMENT_DISABLE !== '0') || + process.env.S3BACKEND === 'mem' + ) { log.info('remote management disabled'); return; } /* Temporary check before to fully move to the process management agent. */ - if (isManagementAgentUsed() ^ typeof callback === 'function') { + if (isManagementAgentUsed() ^ (typeof callback === 'function')) { let msg = 'misuse of initManagement function: '; msg += `MANAGEMENT_USE_AGENT: ${process.env.MANAGEMENT_USE_AGENT}`; msg += `, callback type: ${typeof callback}`; throw new Error(msg); } - async.waterfall([ - // eslint-disable-next-line arrow-body-style - cb => { return isManagementAgentUsed() ? metadata.setup(cb) : cb(); }, - cb => initManagementDatabase(log, cb), - cb => metadata.getUUID(log, cb), - (instanceId, cb) => initManagementCredentials( - managementEndpoint, instanceId, log, cb), - (instanceId, token, cb) => { - if (!isManagementAgentUsed()) { - cb(null, instanceId, token, {}); - return; + async.waterfall( + [ + // eslint-disable-next-line arrow-body-style + cb => { + return isManagementAgentUsed() ? metadata.setup(cb) : cb(); + }, + cb => initManagementDatabase(log, cb), + cb => metadata.getUUID(log, cb), + (instanceId, cb) => initManagementCredentials(managementEndpoint, instanceId, log, cb), + (instanceId, token, cb) => { + if (!isManagementAgentUsed()) { + cb(null, instanceId, token, {}); + return; + } + loadCachedOverlay(log, (err, overlay) => cb(err, instanceId, token, overlay)); + }, + (instanceId, token, overlay, cb) => { + if (!isManagementAgentUsed()) { + cb(null, instanceId, token, overlay); + return; + } + patchConfiguration(overlay, log, err => cb(err, instanceId, token, overlay)); + }, + ], + (error, instanceId, token, overlay) => { + if (error) { + log.error('could not initialize remote management, retrying later', { + error: reshapeExceptionError(error), + method: 'initManagement', + }); + setTimeout(initManagement, initRemoteManagementRetryDelay, logger.newRequestLogger()); + } else { + log.info(`this deployment's Instance ID is ${instanceId}`); + log.end('management init done'); + startManagementListeners(instanceId, token); + if (callback) { + callback(overlay); + } } - loadCachedOverlay(log, (err, overlay) => cb(err, instanceId, - token, overlay)); }, - (instanceId, token, overlay, cb) => { - if (!isManagementAgentUsed()) { - cb(null, instanceId, token, overlay); - return; - } - patchConfiguration(overlay, log, - err => cb(err, instanceId, token, overlay)); - }, - ], (error, instanceId, token, overlay) => { - if (error) { - log.error('could not initialize remote management, retrying later', - { error: reshapeExceptionError(error), - method: 'initManagement' }); - setTimeout(initManagement, - initRemoteManagementRetryDelay, - logger.newRequestLogger()); - } else { - log.info(`this deployment's Instance ID is ${instanceId}`); - log.end('management init done'); - startManagementListeners(instanceId, token); - if (callback) { - callback(overlay); - } - } - }); + ); } module.exports = { diff --git a/lib/management/poll.js b/lib/management/poll.js index 83a72c46c7..9ed3052980 100644 --- a/lib/management/poll.js +++ b/lib/management/poll.js @@ -5,18 +5,13 @@ const request = require('../utilities/request'); const _config = require('../Config').config; const logger = require('../utilities/logger'); const metadata = require('../metadata/wrapper'); -const { - loadCachedOverlay, - patchConfiguration, - saveConfigurationVersion, -} = require('./configuration'); +const { loadCachedOverlay, patchConfiguration, saveConfigurationVersion } = require('./configuration'); const { reshapeExceptionError } = arsenal.errorUtils; const pushReportDelay = 30000; const pullConfigurationOverlayDelay = 60000; -function loadRemoteOverlay( - managementEndpoint, instanceId, remoteToken, cachedOverlay, log, cb) { +function loadRemoteOverlay(managementEndpoint, instanceId, remoteToken, cachedOverlay, log, cb) { log.debug('loading remote overlay'); const opts = { headers: { @@ -25,47 +20,50 @@ function loadRemoteOverlay( }, json: true, }; - request.get(`${managementEndpoint}/${instanceId}/config/overlay`, opts, - (error, response, body) => { - if (error) { - return cb(error); - } - if (response.statusCode === 200) { - return cb(null, cachedOverlay, body); - } - if (response.statusCode === 404) { - return cb(null, cachedOverlay, {}); - } - return cb(arsenal.errors.AccessForbidden, cachedOverlay, {}); - }); -} - -// TODO save only after successful patch -function applyConfigurationOverlay( - managementEndpoint, instanceId, remoteToken, log) { - async.waterfall([ - wcb => loadCachedOverlay(log, wcb), - (cachedOverlay, wcb) => patchConfiguration(cachedOverlay, - log, wcb), - (cachedOverlay, wcb) => - loadRemoteOverlay(managementEndpoint, instanceId, remoteToken, - cachedOverlay, log, wcb), - (cachedOverlay, remoteOverlay, wcb) => - saveConfigurationVersion(cachedOverlay, remoteOverlay, log, wcb), - (remoteOverlay, wcb) => patchConfiguration(remoteOverlay, - log, wcb), - ], error => { + request.get(`${managementEndpoint}/${instanceId}/config/overlay`, opts, (error, response, body) => { if (error) { - log.error('could not apply managed configuration', - { error: reshapeExceptionError(error), - method: 'applyConfigurationOverlay' }); + return cb(error); + } + if (response.statusCode === 200) { + return cb(null, cachedOverlay, body); } - setTimeout(applyConfigurationOverlay, pullConfigurationOverlayDelay, - managementEndpoint, instanceId, remoteToken, - logger.newRequestLogger()); + if (response.statusCode === 404) { + return cb(null, cachedOverlay, {}); + } + return cb(arsenal.errors.AccessForbidden, cachedOverlay, {}); }); } +// TODO save only after successful patch +function applyConfigurationOverlay(managementEndpoint, instanceId, remoteToken, log) { + async.waterfall( + [ + wcb => loadCachedOverlay(log, wcb), + (cachedOverlay, wcb) => patchConfiguration(cachedOverlay, log, wcb), + (cachedOverlay, wcb) => + loadRemoteOverlay(managementEndpoint, instanceId, remoteToken, cachedOverlay, log, wcb), + (cachedOverlay, remoteOverlay, wcb) => saveConfigurationVersion(cachedOverlay, remoteOverlay, log, wcb), + (remoteOverlay, wcb) => patchConfiguration(remoteOverlay, log, wcb), + ], + error => { + if (error) { + log.error('could not apply managed configuration', { + error: reshapeExceptionError(error), + method: 'applyConfigurationOverlay', + }); + } + setTimeout( + applyConfigurationOverlay, + pullConfigurationOverlayDelay, + managementEndpoint, + instanceId, + remoteToken, + logger.newRequestLogger(), + ); + }, + ); +} + function postStats(managementEndpoint, instanceId, remoteToken, report, next) { const toURL = `${managementEndpoint}/${instanceId}/stats`; const toOptions = { @@ -115,18 +113,11 @@ function pushStats(managementEndpoint, instanceId, remoteToken, next) { } logger.debug('report', { report }); - postStats( - managementEndpoint, - instanceId, - remoteToken, - report, - next - ); + postStats(managementEndpoint, instanceId, remoteToken, report, next); return; }); - setTimeout(pushStats, pushReportDelay, - managementEndpoint, instanceId, remoteToken); + setTimeout(pushStats, pushReportDelay, managementEndpoint, instanceId, remoteToken); } /** @@ -141,15 +132,13 @@ function pushStats(managementEndpoint, instanceId, remoteToken, next) { * * @returns {undefined} */ -function startPollingManagementClient( - managementEndpoint, instanceId, remoteToken) { +function startPollingManagementClient(managementEndpoint, instanceId, remoteToken) { metadata.notifyBucketChange(() => { pushStats(managementEndpoint, instanceId, remoteToken); }); pushStats(managementEndpoint, instanceId, remoteToken); - applyConfigurationOverlay(managementEndpoint, instanceId, remoteToken, - logger.newRequestLogger()); + applyConfigurationOverlay(managementEndpoint, instanceId, remoteToken, logger.newRequestLogger()); } module.exports = { diff --git a/lib/management/push.js b/lib/management/push.js index 82ad9be7b5..01123ec376 100644 --- a/lib/management/push.js +++ b/lib/management/push.js @@ -14,25 +14,15 @@ const metadata = require('../metadata/wrapper'); const { reshapeExceptionError } = arsenal.errorUtils; const { isManagementAgentUsed } = require('./agentClient'); const { applyAndSaveOverlay } = require('./configuration'); -const { - ChannelMessageV0, - MessageType, -} = require('./ChannelMessageV0'); - -const { - CONFIG_OVERLAY_MESSAGE, - METRICS_REQUEST_MESSAGE, - CHANNEL_CLOSE_MESSAGE, - CHANNEL_PAYLOAD_MESSAGE, -} = MessageType; +const { ChannelMessageV0, MessageType } = require('./ChannelMessageV0'); + +const { CONFIG_OVERLAY_MESSAGE, METRICS_REQUEST_MESSAGE, CHANNEL_CLOSE_MESSAGE, CHANNEL_PAYLOAD_MESSAGE } = MessageType; const PING_INTERVAL_MS = 10000; const subprotocols = [ChannelMessageV0.protocolName]; -const cloudServerHost = process.env.SECURE_CHANNEL_DEFAULT_FORWARD_TO_HOST - || 'localhost'; -const cloudServerPort = process.env.SECURE_CHANNEL_DEFAULT_FORWARD_TO_PORT - || _config.port; +const cloudServerHost = process.env.SECURE_CHANNEL_DEFAULT_FORWARD_TO_HOST || 'localhost'; +const cloudServerPort = process.env.SECURE_CHANNEL_DEFAULT_FORWARD_TO_PORT || _config.port; let overlayMessageListener = null; let connected = false; @@ -40,8 +30,7 @@ let connected = false; // No wildcard nor cidr/mask match for now function createWSAgent(pushEndpoint, env, log) { const url = new _URL(pushEndpoint); - const noProxy = (env.NO_PROXY || env.no_proxy - || '').split(','); + const noProxy = (env.NO_PROXY || env.no_proxy || '').split(','); if (noProxy.includes(url.hostname)) { log.info('push server ws has proxy exclusion', { noProxy }); @@ -49,20 +38,20 @@ function createWSAgent(pushEndpoint, env, log) { } if (url.protocol === 'https:' || url.protocol === 'wss:') { - const httpsProxy = (env.HTTPS_PROXY || env.https_proxy); + const httpsProxy = env.HTTPS_PROXY || env.https_proxy; if (httpsProxy) { log.info('push server ws using https proxy', { httpsProxy }); return new HttpsProxyAgent(httpsProxy); } } else if (url.protocol === 'http:' || url.protocol === 'ws:') { - const httpProxy = (env.HTTP_PROXY || env.http_proxy); + const httpProxy = env.HTTP_PROXY || env.http_proxy; if (httpProxy) { log.info('push server ws using http proxy', { httpProxy }); return new HttpsProxyAgent(httpProxy); } } - const allProxy = (env.ALL_PROXY || env.all_proxy); + const allProxy = env.ALL_PROXY || env.all_proxy; if (allProxy) { log.info('push server ws using wildcard proxy', { allProxy }); return new HttpsProxyAgent(allProxy); @@ -88,8 +77,7 @@ function startWSManagementClient(url, token, cb) { logger.info('connecting to push server', { url }); function _logError(error, errorMessage, method) { if (error) { - logger.error(`management client error: ${errorMessage}`, - { error: reshapeExceptionError(error), method }); + logger.error(`management client error: ${errorMessage}`, { error: reshapeExceptionError(error), method }); } } @@ -131,9 +119,9 @@ function startWSManagementClient(url, token, cb) { _logError(err, 'failed to get metrics report', 'pushStats'); return; } - ws.send(ChannelMessageV0.encodeMetricsReportMessage(body), - err => _logError(err, 'failed to send metrics report message', - 'pushStats')); + ws.send(ChannelMessageV0.encodeMetricsReportMessage(body), err => + _logError(err, 'failed to send metrics report message', 'pushStats'), + ); }); } @@ -151,17 +139,14 @@ function startWSManagementClient(url, token, cb) { socket = net.createConnection(cloudServerPort, cloudServerHost); socket.on('data', data => { - ws.send(ChannelMessageV0. - encodeChannelDataMessage(channelId, data), err => - _logError(err, 'failed to send channel data message', - 'receiveChannelData')); + ws.send(ChannelMessageV0.encodeChannelDataMessage(channelId, data), err => + _logError(err, 'failed to send channel data message', 'receiveChannelData'), + ); }); - socket.on('connect', () => { - }); + socket.on('connect', () => {}); - socket.on('drain', () => { - }); + socket.on('drain', () => {}); socket.on('error', error => { logger.error('failed to connect to S3', { @@ -174,10 +159,9 @@ function startWSManagementClient(url, token, cb) { socket.on('end', () => { socket.destroy(); socketsByChannelId[channelId] = null; - ws.send(ChannelMessageV0.encodeChannelCloseMessage(channelId), - err => _logError(err, - 'failed to send channel close message', - 'receiveChannelData')); + ws.send(ChannelMessageV0.encodeChannelCloseMessage(channelId), err => + _logError(err, 'failed to send channel close message', 'receiveChannelData'), + ); }); socketsByChannelId[channelId] = socket; @@ -208,8 +192,7 @@ function startWSManagementClient(url, token, cb) { ws.on('close', () => { logger.info('disconnected from push server, reconnecting in 10s'); metadata.notifyBucketChange(null); - _config.removeListener('browser-access-enabled-change', - browserAccessChangeHandler); + _config.removeListener('browser-access-enabled-change', browserAccessChangeHandler); setTimeout(startWSManagementClient, 10000, url, token); connected = false; @@ -241,31 +224,29 @@ function startWSManagementClient(url, token, cb) { const log = logger.newRequestLogger(); const message = new ChannelMessageV0(data); switch (message.getType()) { - case CONFIG_OVERLAY_MESSAGE: - if (!isManagementAgentUsed()) { - applyAndSaveOverlay(JSON.parse(message.getPayload()), log); - } else { - if (overlayMessageListener) { - overlayMessageListener(message.getPayload().toString()); + case CONFIG_OVERLAY_MESSAGE: + if (!isManagementAgentUsed()) { + applyAndSaveOverlay(JSON.parse(message.getPayload()), log); + } else { + if (overlayMessageListener) { + overlayMessageListener(message.getPayload().toString()); + } } - } - break; - case METRICS_REQUEST_MESSAGE: - pushStats(); - break; - case CHANNEL_CLOSE_MESSAGE: - closeChannel(message.getChannelNumber()); - break; - case CHANNEL_PAYLOAD_MESSAGE: - // browserAccessEnabled defaults to true unless explicitly false - if (_config.browserAccessEnabled !== false) { - receiveChannelData( - message.getChannelNumber(), message.getPayload()); - } - break; - default: - logger.error('unknown message type from push server', - { messageType: message.getType() }); + break; + case METRICS_REQUEST_MESSAGE: + pushStats(); + break; + case CHANNEL_CLOSE_MESSAGE: + closeChannel(message.getChannelNumber()); + break; + case CHANNEL_PAYLOAD_MESSAGE: + // browserAccessEnabled defaults to true unless explicitly false + if (_config.browserAccessEnabled !== false) { + receiveChannelData(message.getChannelNumber(), message.getPayload()); + } + break; + default: + logger.error('unknown message type from push server', { messageType: message.getType() }); } }); } diff --git a/lib/metadata/acl.js b/lib/metadata/acl.js index f48ab7aa42..961b95d79f 100644 --- a/lib/metadata/acl.js +++ b/lib/metadata/acl.js @@ -31,14 +31,16 @@ const acl = { * contain the same number of elements, and all elements from one * grant are incuded in the other grant */ - return oldAcl[grant].length === newAcl[grant].length - && oldAcl[grant].every(value => newAcl[grant].includes(value)); + return ( + oldAcl[grant].length === newAcl[grant].length && oldAcl[grant].every(value => newAcl[grant].includes(value)) + ); }, addObjectACL(bucket, objectKey, objectMD, addACLParams, params, log, cb) { log.trace('updating object acl in metadata'); - const isAclUnchanged = Object.keys(objectMD.acl).length === Object.keys(addACLParams).length - && Object.keys(objectMD.acl).every(grant => this._aclGrantDidNotChange(grant, objectMD.acl, addACLParams)); + const isAclUnchanged = + Object.keys(objectMD.acl).length === Object.keys(addACLParams).length && + Object.keys(objectMD.acl).every(grant => this._aclGrantDidNotChange(grant, objectMD.acl, addACLParams)); if (!isAclUnchanged) { /* eslint-disable no-param-reassign */ objectMD.acl = addACLParams; @@ -77,14 +79,22 @@ const acl = { }; let validCannedACL = []; if (resourceType === 'bucket') { - validCannedACL = - ['private', 'public-read', 'public-read-write', - 'authenticated-read', 'log-delivery-write']; + validCannedACL = [ + 'private', + 'public-read', + 'public-read-write', + 'authenticated-read', + 'log-delivery-write', + ]; } else if (resourceType === 'object') { - validCannedACL = - ['private', 'public-read', 'public-read-write', - 'authenticated-read', 'bucket-owner-read', - 'bucket-owner-full-control']; + validCannedACL = [ + 'private', + 'public-read', + 'public-read-write', + 'authenticated-read', + 'bucket-owner-read', + 'bucket-owner-full-control', + ]; } // parse canned acl @@ -98,45 +108,34 @@ const acl = { } // parse grant headers - const grantReadHeader = - aclUtils.parseGrant(headers['x-amz-grant-read'], 'READ'); + const grantReadHeader = aclUtils.parseGrant(headers['x-amz-grant-read'], 'READ'); let grantWriteHeader = []; if (resourceType === 'bucket') { - grantWriteHeader = aclUtils - .parseGrant(headers['x-amz-grant-write'], 'WRITE'); + grantWriteHeader = aclUtils.parseGrant(headers['x-amz-grant-write'], 'WRITE'); } - const grantReadACPHeader = aclUtils - .parseGrant(headers['x-amz-grant-read-acp'], 'READ_ACP'); - const grantWriteACPHeader = aclUtils - .parseGrant(headers['x-amz-grant-write-acp'], 'WRITE_ACP'); - const grantFullControlHeader = aclUtils - .parseGrant(headers['x-amz-grant-full-control'], 'FULL_CONTROL'); - const allGrantHeaders = - [].concat(grantReadHeader, grantWriteHeader, - grantReadACPHeader, grantWriteACPHeader, - grantFullControlHeader).filter(item => item !== undefined); + const grantReadACPHeader = aclUtils.parseGrant(headers['x-amz-grant-read-acp'], 'READ_ACP'); + const grantWriteACPHeader = aclUtils.parseGrant(headers['x-amz-grant-write-acp'], 'WRITE_ACP'); + const grantFullControlHeader = aclUtils.parseGrant(headers['x-amz-grant-full-control'], 'FULL_CONTROL'); + const allGrantHeaders = [] + .concat(grantReadHeader, grantWriteHeader, grantReadACPHeader, grantWriteACPHeader, grantFullControlHeader) + .filter(item => item !== undefined); if (allGrantHeaders.length === 0) { return cb(null, currentResourceACL); } - const usersIdentifiedByEmail = allGrantHeaders - .filter(it => it && it.userIDType.toLowerCase() === 'emailaddress'); - const usersIdentifiedByGroup = allGrantHeaders - .filter(item => item && item.userIDType.toLowerCase() === 'uri'); + const usersIdentifiedByEmail = allGrantHeaders.filter( + it => it && it.userIDType.toLowerCase() === 'emailaddress', + ); + const usersIdentifiedByGroup = allGrantHeaders.filter(item => item && item.userIDType.toLowerCase() === 'uri'); const justEmails = usersIdentifiedByEmail.map(item => item.identifier); - const validGroups = [ - constants.allAuthedUsersId, - constants.publicId, - constants.logId, - ]; + const validGroups = [constants.allAuthedUsersId, constants.publicId, constants.logId]; for (let i = 0; i < usersIdentifiedByGroup.length; i++) { if (validGroups.indexOf(usersIdentifiedByGroup[i].identifier) < 0) { return cb(errors.InvalidArgument); } } - const usersIdentifiedByID = allGrantHeaders - .filter(item => item && item.userIDType.toLowerCase() === 'id'); + const usersIdentifiedByID = allGrantHeaders.filter(item => item && item.userIDType.toLowerCase() === 'id'); // TODO: Consider whether want to verify with Vault // whether canonicalID is associated with existing // account before adding to ACL @@ -148,22 +147,22 @@ const acl = { if (err) { return cb(err); } - const reconstructedUsersIdentifiedByEmail = aclUtils. - reconstructUsersIdentifiedByEmail(results, - usersIdentifiedByEmail); + const reconstructedUsersIdentifiedByEmail = aclUtils.reconstructUsersIdentifiedByEmail( + results, + usersIdentifiedByEmail, + ); const allUsers = [].concat( reconstructedUsersIdentifiedByEmail, usersIdentifiedByGroup, - usersIdentifiedByID); - const revisedACL = - aclUtils.sortHeaderGrants(allUsers, resourceACL); + usersIdentifiedByID, + ); + const revisedACL = aclUtils.sortHeaderGrants(allUsers, resourceACL); return cb(null, revisedACL); }); } else { // If don't have to look up canonicalID's just sort grants // and add to bucket - const revisedACL = aclUtils - .sortHeaderGrants(allGrantHeaders, resourceACL); + const revisedACL = aclUtils.sortHeaderGrants(allGrantHeaders, resourceACL); return cb(null, revisedACL); } return undefined; @@ -171,4 +170,3 @@ const acl = { }; module.exports = acl; - diff --git a/lib/metadata/wrapper.js b/lib/metadata/wrapper.js index 6bd800e60f..947d103847 100644 --- a/lib/metadata/wrapper.js +++ b/lib/metadata/wrapper.js @@ -39,6 +39,5 @@ if (clientName === 'mem') { }; } -const metadata = new MetadataWrapper(config.backends.metadata, params, - bucketclient, logger); +const metadata = new MetadataWrapper(config.backends.metadata, params, bucketclient, logger); module.exports = metadata; diff --git a/lib/nfs/utilities.js b/lib/nfs/utilities.js index d8d0a5e9f3..83a80146db 100644 --- a/lib/nfs/utilities.js +++ b/lib/nfs/utilities.js @@ -4,8 +4,7 @@ const http = require('http'); const https = require('https'); const logger = require('../utilities/logger'); -function _createBucketWithNFSEnabled(host, port, bucketName, accessKey, - secretKey, verbose, ssl, locationConstraint) { +function _createBucketWithNFSEnabled(host, port, bucketName, accessKey, secretKey, verbose, ssl, locationConstraint) { const options = { host, port, @@ -47,10 +46,11 @@ function _createBucketWithNFSEnabled(host, port, bucketName, accessKey, logger.info('request headers', { headers: request.getHeaders() }); } if (locationConstraint) { - const createBucketConfiguration = '' + - `${locationConstraint}` + - ''; + const createBucketConfiguration = + '' + + `${locationConstraint}` + + ''; request.write(createBucketConfiguration); } request.end(); @@ -73,20 +73,17 @@ function createBucketWithNFSEnabled() { .option('-p, --port ', 'Port of the server') .option('-s', '--ssl', 'Enable ssl') .option('-v, --verbose') - .option('-l, --location-constraint ', - 'location Constraint') + .option('-l, --location-constraint ', 'location Constraint') .parse(process.argv); - const { host, port, accessKey, secretKey, bucket, verbose, - ssl, locationConstraint } = commander; + const { host, port, accessKey, secretKey, bucket, verbose, ssl, locationConstraint } = commander; if (!host || !port || !accessKey || !secretKey || !bucket) { logger.error('missing parameter'); commander.outputHelp(); process.exit(1); } - _createBucketWithNFSEnabled(host, port, bucket, accessKey, secretKey, - verbose, ssl, locationConstraint); + _createBucketWithNFSEnabled(host, port, bucket, accessKey, secretKey, verbose, ssl, locationConstraint); } module.exports = { diff --git a/lib/routes/routeMetadata.js b/lib/routes/routeMetadata.js index a5cac213e2..64319b3c6e 100644 --- a/lib/routes/routeMetadata.js +++ b/lib/routes/routeMetadata.js @@ -45,68 +45,85 @@ function routeMetadata(clientIP, request, response, log) { // restrict access to only routes ending in bucket, log or id const { resourceType, subResource } = request; - if (resourceType === 'admin' - && !['bucket', 'log', 'id'].includes(subResource)) { + if (resourceType === 'admin' && !['bucket', 'log', 'id'].includes(subResource)) { return responseJSONBody(errors.NotImplemented, null, response, log); } const ip = requestUtils.getClientIp(request, config); const isSecure = requestUtils.getHttpProtocolSecurity(request, config); - const requestContexts = [new RequestContext(request.headers, request.query, - request.generalResource, request.specificResource, ip, - isSecure, request.resourceType, 'metadata')]; - return waterfall([ - next => auth.server.doAuth(request, log, (err, userInfo, authRes) => { - if (err) { - log.debug('authentication error', { - error: err, - method: request.method, - bucketName: request.bucketName, - objectKey: request.objectKey, - }); - return next(err); - } - // authRes is not defined for account credentials - if (authRes && !authRes[0].isAllowed) { - return next(errors.AccessDenied); - } - return next(null, userInfo); - }, 's3', requestContexts), - (userInfo, next) => { - if (userInfo.getCanonicalID() === constants.publicId) { - log.debug('unauthenticated access to API routes', { - method: request.method, - bucketName: request.bucketName, - objectKey: request.objectKey, - }); - return next(errors.AccessDenied); - } - const { url } = request; - const path = url.startsWith('/_/metadata/admin') ? - url.replace('/_/metadata/admin/', '/_/') : - url.replace('/_/metadata/', '/'); - // bucketd is always configured on the loopback interface in s3c - const endpoint = bootstrap[0]; - const target = `http://${endpoint}${path}`; - return metadataProxy.web(request, response, { target }, err => { - if (err) { - log.error('error proxying request to metadata admin server', - { error: err.message }); - return next(errors.ServiceUnavailable); + const requestContexts = [ + new RequestContext( + request.headers, + request.query, + request.generalResource, + request.specificResource, + ip, + isSecure, + request.resourceType, + 'metadata', + ), + ]; + return waterfall( + [ + next => + auth.server.doAuth( + request, + log, + (err, userInfo, authRes) => { + if (err) { + log.debug('authentication error', { + error: err, + method: request.method, + bucketName: request.bucketName, + objectKey: request.objectKey, + }); + return next(err); + } + // authRes is not defined for account credentials + if (authRes && !authRes[0].isAllowed) { + return next(errors.AccessDenied); + } + return next(null, userInfo); + }, + 's3', + requestContexts, + ), + (userInfo, next) => { + if (userInfo.getCanonicalID() === constants.publicId) { + log.debug('unauthenticated access to API routes', { + method: request.method, + bucketName: request.bucketName, + objectKey: request.objectKey, + }); + return next(errors.AccessDenied); } - return next(); - }); - }], + const { url } = request; + const path = url.startsWith('/_/metadata/admin') + ? url.replace('/_/metadata/admin/', '/_/') + : url.replace('/_/metadata/', '/'); + // bucketd is always configured on the loopback interface in s3c + const endpoint = bootstrap[0]; + const target = `http://${endpoint}${path}`; + return metadataProxy.web(request, response, { target }, err => { + if (err) { + log.error('error proxying request to metadata admin server', { error: err.message }); + return next(errors.ServiceUnavailable); + } + return next(); + }); + }, + ], err => { if (err) { return responseJSONBody(err, null, response, log); } - log.debug('metadata route response sent successfully', - { method: request.method, - bucketName: request.bucketName, - objectKey: request.objectKey }); + log.debug('metadata route response sent successfully', { + method: request.method, + bucketName: request.bucketName, + objectKey: request.objectKey, + }); return undefined; - }); + }, + ); } - module.exports = routeMetadata; diff --git a/lib/routes/routeVeeam.js b/lib/routes/routeVeeam.js index bfd83ae1e5..b6dc8f4f80 100644 --- a/lib/routes/routeVeeam.js +++ b/lib/routes/routeVeeam.js @@ -16,10 +16,7 @@ const { responseXMLBody } = s3routes.routesUtils; auth.setHandler(vault); -const validObjectKeys = [ - `${validPath}system.xml`, - `${validPath}capacity.xml`, -]; +const validObjectKeys = [`${validPath}system.xml`, `${validPath}capacity.xml`]; const apiToAction = { PUT: 'PutObject', @@ -29,10 +26,7 @@ const apiToAction = { LIST: 'ListObjects', }; -const allowedSdkQueryKeys = new Set([ - 'x-id', - 'x-amz-user-agent', -]); +const allowedSdkQueryKeys = new Set(['x-id', 'x-amz-user-agent']); // Allowed query parameters for SigV4 presigned URLs (lower-cased). const allowedPresignQueryKeys = new Set([ @@ -91,17 +85,18 @@ function checkBucketAndKey(bucketName, objectKey, requestQueryParams, method, lo // Ensure x-id, when present, matches the expected action for the method. if (normalizedKey === 'x-id' && value !== apiToAction[method]) { - return errorInstances.InvalidRequest - .customizeDescription('The Veeam SOSAPI folder does not support this action.'); + return errorInstances.InvalidRequest.customizeDescription( + 'The Veeam SOSAPI folder does not support this action.', + ); } - const isAllowedSdkKey = allowedSdkQueryKeys.has(normalizedKey) - || normalizedKey.startsWith('x-amz-sdk-'); + const isAllowedSdkKey = allowedSdkQueryKeys.has(normalizedKey) || normalizedKey.startsWith('x-amz-sdk-'); const isAllowedPresignKey = allowedPresignQueryKeys.has(normalizedKey); if (!isAllowedSdkKey && !isAllowedPresignKey) { - return errorInstances.InvalidRequest - .customizeDescription('The Veeam SOSAPI folder does not support this action.'); + return errorInstances.InvalidRequest.customizeDescription( + 'The Veeam SOSAPI folder does not support this action.', + ); } } if (typeof objectKey !== 'string' || !validObjectKeys.includes(objectKey)) { @@ -128,44 +123,54 @@ function authorizationMiddleware(request, response, api, log, callback) { return responseXMLBody(errors.AccessDenied, null, response, log); } const requestContexts = prepareRequestContexts(api, request); - return async.waterfall([ - next => auth.server.doAuth(request, log, (err, userInfo, authorizationResults, streamingV4Params) => { - if (err) { - log.debug('authentication error', { - error: err, - method: request.method, + return async.waterfall( + [ + next => + auth.server.doAuth( + request, + log, + (err, userInfo, authorizationResults, streamingV4Params) => { + if (err) { + log.debug('authentication error', { + error: err, + method: request.method, + bucketName: request.bucketName, + objectKey: request.objectKey, + }); + } + /* eslint-disable no-param-reassign */ + request.authorizationResults = authorizationResults; + request.streamingV4Params = streamingV4Params; + /* eslint-enable no-param-reassign */ + return next(err, userInfo); + }, + 's3', + requestContexts, + ), + (userInfo, next) => { + // Ensure only supported HTTP verbs and actions are called, + // otherwise deny access + const requestType = apiToAction[api]; + if (!requestType) { + return next(errors.AccessDenied); + } + const mdValParams = { bucketName: request.bucketName, - objectKey: request.objectKey, - }); - } - /* eslint-disable no-param-reassign */ - request.authorizationResults = authorizationResults; - request.streamingV4Params = streamingV4Params; - /* eslint-enable no-param-reassign */ - return next(err, userInfo); - }, 's3', requestContexts), - (userInfo, next) => { - // Ensure only supported HTTP verbs and actions are called, - // otherwise deny access - const requestType = apiToAction[api]; - if (!requestType) { - return next(errors.AccessDenied); + authInfo: userInfo, + requestType, + request, + }; + return next(null, mdValParams); + }, + (mdValParams, next) => standardMetadataValidateBucket(mdValParams, request.actionImplicitDenies, log, next), + ], + (err, bucketMd) => { + if (err || !bucketMd) { + return responseXMLBody(err, null, response, log); } - const mdValParams = { - bucketName: request.bucketName, - authInfo: userInfo, - requestType, - request, - }; - return next(null, mdValParams); + return callback(request, response, bucketMd, log); }, - (mdValParams, next) => standardMetadataValidateBucket(mdValParams, request.actionImplicitDenies, log, next), - ], (err, bucketMd) => { - if (err || !bucketMd) { - return responseXMLBody(err, null, response, log); - } - return callback(request, response, bucketMd, log); - }); + ); } function _normalizeVeeamRequest(req) { @@ -183,11 +188,10 @@ function _normalizeVeeamRequest(req) { req.query = parsedUrl.query; req.bucketName = pathArr[1]; req.objectKey = pathArr.slice(2).join('/'); - const contentLength = req.headers['x-amz-decoded-content-length'] ? - req.headers['x-amz-decoded-content-length'] : - req.headers['content-length']; - req.parsedContentLength = - Number.parseInt(contentLength?.toString() ?? '', 10); + const contentLength = req.headers['x-amz-decoded-content-length'] + ? req.headers['x-amz-decoded-content-length'] + : req.headers['content-length']; + req.parsedContentLength = Number.parseInt(contentLength?.toString() ?? '', 10); /* eslint-enable no-param-reassign */ } @@ -237,11 +241,15 @@ function routeVeeam(clientIP, request, response, log) { return responseXMLBody(error, '', response, log); } const bucketOrKeyError = checkBucketAndKey( - request.bucketName, request.objectKey, request.query, requestMethod, log); + request.bucketName, + request.objectKey, + request.query, + requestMethod, + log, + ); if (bucketOrKeyError) { - log.error('error with bucket or key value', - { error: bucketOrKeyError }); + log.error('error with bucket or key value', { error: bucketOrKeyError }); return routesUtils.responseXMLBody(bucketOrKeyError, null, response, log); } return authorizationMiddleware(request, response, requestMethod, log, method); diff --git a/lib/routes/routeWorkflowEngineOperator.js b/lib/routes/routeWorkflowEngineOperator.js index bf73a1ed68..c231cf9e47 100644 --- a/lib/routes/routeWorkflowEngineOperator.js +++ b/lib/routes/routeWorkflowEngineOperator.js @@ -4,12 +4,10 @@ const httpProxy = require('http-proxy'); const workflowEngineOperatorProxy = httpProxy.createProxyServer({ ignorePath: true, }); -const { auth, errors, s3routes } = - require('arsenal'); +const { auth, errors, s3routes } = require('arsenal'); const { responseJSONBody } = s3routes.routesUtils; const vault = require('../auth/vault'); -const prepareRequestContexts = require( -'../api/apiUtils/authorization/prepareRequestContexts'); +const prepareRequestContexts = require('../api/apiUtils/authorization/prepareRequestContexts'); const { config } = require('../Config'); const constants = require('../../constants'); @@ -52,45 +50,45 @@ function routeWorkflowEngineOperator(clientIP, request, response, log) { log.debug('unable to proxy workflow engine operator request', { workflowEngineConfig: config.workflowEngineOperator, }); - return responseJSONBody(errors.MethodNotAllowed, null, response, - log); + return responseJSONBody(errors.MethodNotAllowed, null, response, log); } const path = request.url.replace('/_/workflow-engine-operator/api', '/_/'); const { host, port } = config.workflowEngineOperator; const target = `http://${host}:${port}${path}`; - return auth.server.doAuth(request, log, (err, userInfo) => { - if (err) { - log.debug('authentication error', { - error: err, - method: request.method, - bucketName: request.bucketName, - objectKey: request.objectKey, + return auth.server.doAuth( + request, + log, + (err, userInfo) => { + if (err) { + log.debug('authentication error', { + error: err, + method: request.method, + bucketName: request.bucketName, + objectKey: request.objectKey, + }); + return responseJSONBody(err, null, response, log); + } + // FIXME for now, any authenticated user can access API + // routes. We should introduce admin accounts or accounts + // with admin privileges, and restrict access to those + // only. + if (userInfo.getCanonicalID() === constants.publicId) { + log.debug('unauthenticated access to API routes', { + method: request.method, + bucketName: request.bucketName, + objectKey: request.objectKey, + }); + return responseJSONBody(errors.AccessDenied, null, response, log); + } + return workflowEngineOperatorProxy.web(request, response, { target }, err => { + log.error('error proxying request to api server', { error: err.message }); + return responseJSONBody(errors.ServiceUnavailable, null, response, log); }); - return responseJSONBody(err, null, response, log); - } - // FIXME for now, any authenticated user can access API - // routes. We should introduce admin accounts or accounts - // with admin privileges, and restrict access to those - // only. - if (userInfo.getCanonicalID() === constants.publicId) { - log.debug('unauthenticated access to API routes', { - method: request.method, - bucketName: request.bucketName, - objectKey: request.objectKey, - }); - return responseJSONBody( - errors.AccessDenied, null, response, log); - } - return workflowEngineOperatorProxy.web( - request, response, { target }, err => { - log.error('error proxying request to api server', - { error: err.message }); - return responseJSONBody(errors.ServiceUnavailable, null, - response, log); - }); - }, 's3', requestContexts); + }, + 's3', + requestContexts, + ); } } - module.exports = routeWorkflowEngineOperator; diff --git a/lib/routes/utilities/pushReplicationMetric.js b/lib/routes/utilities/pushReplicationMetric.js index c81027a494..1fb95f24c3 100644 --- a/lib/routes/utilities/pushReplicationMetric.js +++ b/lib/routes/utilities/pushReplicationMetric.js @@ -14,10 +14,7 @@ function getMetricToPush(prevObjectMD, newObjectMD) { // metrics if their value has changed. try { assert.deepStrictEqual(prevObjectMD.getAcl(), newObjectMD.getAcl()); - assert.deepStrictEqual( - prevObjectMD.getTags(), - newObjectMD.getTags() - ); + assert.deepStrictEqual(prevObjectMD.getTags(), newObjectMD.getTags()); } catch { return 'replicateTags'; } diff --git a/lib/routes/veeam/delete.js b/lib/routes/veeam/delete.js index 20d73a1662..cfdba191c9 100644 --- a/lib/routes/veeam/delete.js +++ b/lib/routes/veeam/delete.js @@ -1,4 +1,3 @@ - const { s3routes, errors } = require('arsenal'); const metadata = require('../../metadata/wrapper'); const { isSystemXML } = require('./utils'); @@ -18,8 +17,7 @@ function deleteVeeamCapabilities(bucketName, objectKey, bucketMd, log, callback) const capabilityFieldName = isSystemXML(objectKey) ? 'SystemInfo' : 'CapacityInfo'; // Ensure file exists in metadata before deletion - if (!bucketMd._capabilities?.VeeamSOSApi - || !bucketMd._capabilities?.VeeamSOSApi[capabilityFieldName]) { + if (!bucketMd._capabilities?.VeeamSOSApi || !bucketMd._capabilities?.VeeamSOSApi[capabilityFieldName]) { return callback(errors.NoSuchKey); } // eslint-disable-next-line no-param-reassign diff --git a/lib/routes/veeam/get.js b/lib/routes/veeam/get.js index 213d5c4866..0159cd18b2 100644 --- a/lib/routes/veeam/get.js +++ b/lib/routes/veeam/get.js @@ -17,8 +17,13 @@ async function getVeeamFile(request, response, bucketMd, log) { } if ('tagging' in request.query) { - return await respondWithData(request, response, log, bucketMd, - buildHeadXML('')); + return await respondWithData( + request, + response, + log, + bucketMd, + buildHeadXML(''), + ); } try { diff --git a/lib/routes/veeam/list.js b/lib/routes/veeam/list.js index 175fbb77b4..a4a1bb5763 100644 --- a/lib/routes/veeam/list.js +++ b/lib/routes/veeam/list.js @@ -44,7 +44,7 @@ function buildXMLResponse(request, arrayOfFiles, versioned = false) { DisplayName: 'Veeam SOSAPI', }, StorageClass: 'VIRTUAL', - } + }, })); entries.push({ key: validPath, @@ -59,7 +59,7 @@ function buildXMLResponse(request, arrayOfFiles, versioned = false) { DisplayName: 'Veeam SOSAPI', }, StorageClass: 'VIRTUAL', - } + }, }); // Add the folder as the base file if (versioned) { @@ -87,8 +87,12 @@ async function listVeeamFiles(request, response, bucketMd, log) { // Only accept list-type query parameter if (!('list-type' in request.query) && !('versions' in request.query)) { - return responseXMLBody(errorInstances.InvalidRequest - .customizeDescription('The Veeam folder does not support this action.'), null, response, log); + return responseXMLBody( + errorInstances.InvalidRequest.customizeDescription('The Veeam folder does not support this action.'), + null, + response, + log, + ); } let data; @@ -125,22 +129,24 @@ async function listVeeamFiles(request, response, bucketMd, log) { } fieldsToGenerate.forEach(file => { const isCapacity = file.name.endsWith('capacity.xml'); - const lastModified = isCapacity - ? bucketMetrics.date - : file.LastModified; + const lastModified = isCapacity ? bucketMetrics.date : file.LastModified; // eslint-disable-next-line no-param-reassign delete file.LastModified; const dataBuffer = Buffer.from(buildXML(file)); filesToBuild.push({ - ...getResponseHeader(request, data, - dataBuffer, lastModified, log), + ...getResponseHeader(request, data, dataBuffer, lastModified, log), name: file.name, }); }); // When `versions` is present, listing should return a versioned list - return await respondWithData(request, response, log, data, - buildXMLResponse(request, filesToBuild, 'versions' in request.query)); + return await respondWithData( + request, + response, + log, + data, + buildXMLResponse(request, filesToBuild, 'versions' in request.query), + ); } module.exports = listVeeamFiles; diff --git a/lib/routes/veeam/put.js b/lib/routes/veeam/put.js index 0e15d86051..d9991775af 100644 --- a/lib/routes/veeam/put.js +++ b/lib/routes/veeam/put.js @@ -25,57 +25,68 @@ function putVeeamFile(request, response, bucketMd, log) { return errors.NoSuchBucket; } - return async.waterfall([ - next => { - // Extract the data from the request, keep it in memory - writeContinue(request, response); - return callbackify(receiveData)(request, log, next); - }, - (value, next) => parseString(value, { explicitArray: false }, (err, parsed) => { - // Convert the received XML to a JS object + return async.waterfall( + [ + next => { + // Extract the data from the request, keep it in memory + writeContinue(request, response); + return callbackify(receiveData)(request, log, next); + }, + (value, next) => + parseString(value, { explicitArray: false }, (err, parsed) => { + // Convert the received XML to a JS object + if (err) { + return next(errors.MalformedXML); + } + return next(null, parsed); + }), + (parsedXML, next) => { + const capabilities = bucketMd._capabilities || { + VeeamSOSApi: {}, + }; + // Validate the JS object schema with joi and prepare the object for + // further logic + const validateFn = isSystemXML(request.objectKey) ? parseSystemSchema : parseCapacitySchema; + let validatedData = null; + try { + validatedData = validateFn(parsedXML); + } catch (err) { + log.error('xml file did not pass validation', { err }); + return next(errors.MalformedXML); + } + const file = getFileToBuild(request, validatedData, true); + if (file.error) { + return next(file.error); + } + capabilities.VeeamSOSApi = { + ...(capabilities.VeeamSOSApi || {}), + ...file.value, + }; + // Write data to bucketMD with the same (validated) format + // eslint-disable-next-line no-param-reassign + bucketMd = { + ...bucketMd, + _capabilities: capabilities, + }; + // Update bucket metadata + return metadata.updateBucketCapabilities( + request.bucketName, + bucketMd, + 'VeeamSOSApi', + file.fieldName, + file.value[file.fieldName], + log, + next, + ); + }, + ], + err => { if (err) { - return next(errors.MalformedXML); - } - return next(null, parsed); - }), - (parsedXML, next) => { - const capabilities = bucketMd._capabilities || { - VeeamSOSApi: {}, - }; - // Validate the JS object schema with joi and prepare the object for - // further logic - const validateFn = isSystemXML(request.objectKey) ? parseSystemSchema : parseCapacitySchema; - let validatedData = null; - try { - validatedData = validateFn(parsedXML); - } catch (err) { - log.error('xml file did not pass validation', { err }); - return next(errors.MalformedXML); + return responseXMLBody(err, null, response, log); } - const file = getFileToBuild(request, validatedData, true); - if (file.error) { - return next(file.error); - } - capabilities.VeeamSOSApi = { - ...(capabilities.VeeamSOSApi || {}), - ...file.value, - }; - // Write data to bucketMD with the same (validated) format - // eslint-disable-next-line no-param-reassign - bucketMd = { - ...bucketMd, - _capabilities: capabilities, - }; - // Update bucket metadata - return metadata.updateBucketCapabilities( - request.bucketName, bucketMd, 'VeeamSOSApi', file.fieldName, file.value[file.fieldName], log, next); - } - ], err => { - if (err) { - return responseXMLBody(err, null, response, log); - } - return responseNoBody(null, null, response, 200, log); - }); + return responseNoBody(null, null, response, 200, log); + }, + ); } module.exports = putVeeamFile; diff --git a/lib/routes/veeam/schemas/capacity.js b/lib/routes/veeam/schemas/capacity.js index 3cd81f7b40..4897b1dae4 100644 --- a/lib/routes/veeam/schemas/capacity.js +++ b/lib/routes/veeam/schemas/capacity.js @@ -18,11 +18,13 @@ const { errors } = require('arsenal'); */ function validateCapacitySchema(parsedXML) { const schema = joi.object({ - CapacityInfo: joi.object({ - Capacity: joi.number().min(-1).integer().required(), - Available: joi.number().min(-1).integer().required(), - Used: joi.number().min(-1).integer().required(), - }).required(), + CapacityInfo: joi + .object({ + Capacity: joi.number().min(-1).integer().required(), + Available: joi.number().min(-1).integer().required(), + Used: joi.number().min(-1).integer().required(), + }) + .required(), }); const validatedData = schema.validate(parsedXML, { // Allow any unknown keys for future compatibility diff --git a/lib/routes/veeam/schemas/system.js b/lib/routes/veeam/schemas/system.js index 400bf22c32..183cf7fd06 100644 --- a/lib/routes/veeam/schemas/system.js +++ b/lib/routes/veeam/schemas/system.js @@ -3,29 +3,31 @@ const { errors, errorInstances } = require('arsenal'); // Allow supporting any version of the protocol const systemSchemasPerVersion = { - 'unsupported': joi.object({}), + unsupported: joi.object({}), '"1.0"': joi.object({ - SystemInfo: joi.object({ - ProtocolVersion: joi.string().required(), - ModelName: joi.string().required(), - ProtocolCapabilities: joi.object({ - CapacityInfo: joi.boolean().required(), - UploadSessions: joi.boolean().required(), - IAMSTS: joi.boolean().default(false), - }).required(), - APIEndpoints: joi.object({ - IAMEndpoint: joi.string().required(), - STSEndpoint: joi.string().required() - }), - SystemRecommendations: joi.object({ - S3ConcurrentTaskLimit: joi.number().min(0).default(64), - S3MultiObjectDeleteLimit: joi.number().min(1).default(1000), - StorageCurrentTasksLimit: joi.number().min(0).default(0), - KbBlockSize: joi.number() - .valid(256, 512, 1024, 2048, 4096, 8192) - .default(1024), - }), - }).required() + SystemInfo: joi + .object({ + ProtocolVersion: joi.string().required(), + ModelName: joi.string().required(), + ProtocolCapabilities: joi + .object({ + CapacityInfo: joi.boolean().required(), + UploadSessions: joi.boolean().required(), + IAMSTS: joi.boolean().default(false), + }) + .required(), + APIEndpoints: joi.object({ + IAMEndpoint: joi.string().required(), + STSEndpoint: joi.string().required(), + }), + SystemRecommendations: joi.object({ + S3ConcurrentTaskLimit: joi.number().min(0).default(64), + S3MultiObjectDeleteLimit: joi.number().min(1).default(1000), + StorageCurrentTasksLimit: joi.number().min(0).default(0), + KbBlockSize: joi.number().valid(256, 512, 1024, 2048, 4096, 8192).default(1024), + }), + }) + .required(), }), }; @@ -62,8 +64,9 @@ function validateSystemSchema(parsedXML) { const protocolVersion = parsedXML?.SystemInfo?.ProtocolVersion; let schema = systemSchemasPerVersion.unsupported; if (!protocolVersion) { - throw new Error(errorInstances.MalformedXML - .customizeDescription('ProtocolVersion must be set for the system.xml file')); + throw new Error( + errorInstances.MalformedXML.customizeDescription('ProtocolVersion must be set for the system.xml file'), + ); } if (protocolVersion && protocolVersion in systemSchemasPerVersion) { schema = systemSchemasPerVersion[parsedXML?.SystemInfo?.ProtocolVersion]; @@ -80,8 +83,10 @@ function validateSystemSchema(parsedXML) { case '"1.0"': // Ensure conditional fields are set // IAMSTS === true implies that SystemInfo.APIEndpoints is defined - if (validatedData.value.SystemInfo.ProtocolCapabilities.IAMSTS - && !validatedData.value.SystemInfo.APIEndpoints) { + if ( + validatedData.value.SystemInfo.ProtocolCapabilities.IAMSTS && + !validatedData.value.SystemInfo.APIEndpoints + ) { throw new Error(errors.MalformedXML); } break; diff --git a/lib/routes/veeam/utils.js b/lib/routes/veeam/utils.js index 3676669435..49ab585ec4 100644 --- a/lib/routes/veeam/utils.js +++ b/lib/routes/veeam/utils.js @@ -37,12 +37,15 @@ async function receiveData(request, log) { // Prevent memory overloads by limiting the size of the // received data. if (parsedContentLength > ContentLengthThreshold) { - throw errorInstances.InvalidInput - .customizeDescription(`maximum allowed content-length is ${ContentLengthThreshold} bytes`); + throw errorInstances.InvalidInput.customizeDescription( + `maximum allowed content-length is ${ContentLengthThreshold} bytes`, + ); } return await new Promise((resolve, reject) => { const settle = jsutil.once((err, result) => { - if (err) { return reject(err); } + if (err) { + return reject(err); + } return resolve(result); }); let totalLength = 0; @@ -51,8 +54,7 @@ async function receiveData(request, log) { write(chunk, _enc, cb) { totalLength += chunk.length; if (totalLength > parsedContentLength) { - log.error('data stream exceed announced size', - { parsedContentLength, overflow: totalLength }); + log.error('data stream exceed announced size', { parsedContentLength, overflow: totalLength }); return cb(errors.InternalError); } chunks.push(chunk); @@ -88,17 +90,18 @@ function buildHeadXML(xmlContent) { * @returns {object} - response headers */ function getResponseHeader(request, bucket, dataBuffer, lastModified, log) { - const corsHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); - const responseMetaHeaders = collectResponseHeaders({ - 'last-modified': lastModified || new Date().toISOString(), - 'content-md5': crypto - .createHash('md5') - .update(dataBuffer) - .digest('hex'), - 'content-length': dataBuffer.byteLength, - 'content-type': 'text/xml', - }, corsHeaders, null, false); + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + const responseMetaHeaders = collectResponseHeaders( + { + 'last-modified': lastModified || new Date().toISOString(), + 'content-md5': crypto.createHash('md5').update(dataBuffer).digest('hex'), + 'content-length': dataBuffer.byteLength, + 'content-type': 'text/xml', + }, + corsHeaders, + null, + false, + ); responseMetaHeaders.versionId = 'null'; responseMetaHeaders['x-amz-id-2'] = log.getSerializedUids(); responseMetaHeaders['x-amz-request-id'] = log.getSerializedUids(); @@ -136,10 +139,10 @@ async function respondWithData(request, response, log, bucket, data, lastModifie try { response.setHeader(key, responseMetaHeaders[key]); } catch (e) { - log.debug('header can not be added ' + - 'to the response', { + log.debug('header can not be added ' + 'to the response', { header: responseMetaHeaders[key], - error: e.stack, method: 'routeVeeam/respondWithData' + error: e.stack, + method: 'routeVeeam/respondWithData', }); } } @@ -199,7 +202,7 @@ function getFileToBuild(request, data, inlineLastModified = false) { return { error: errors.NoSuchKey }; } - const modified = fileToBuild.LastModified || (new Date()).toISOString(); + const modified = fileToBuild.LastModified || new Date().toISOString(); const fieldName = _isSystemXML ? 'SystemInfo' : 'CapacityInfo'; if (inlineLastModified) { @@ -286,9 +289,11 @@ async function buildVeeamFileData(request, bucketMd, log) { } const modified = bucketMetrics.date; - if (bucketMetrics.bytesTotal !== undefined - && fileToBuild.value.CapacityInfo - && !fileToBuild.value.CapacityInfo.Used) { + if ( + bucketMetrics.bytesTotal !== undefined && + fileToBuild.value.CapacityInfo && + !fileToBuild.value.CapacityInfo.Used + ) { fileToBuild.value.CapacityInfo.Used = Number(bucketMetrics.bytesTotal); fileToBuild.value.CapacityInfo.Available = Number(fileToBuild.value.CapacityInfo.Capacity) - Number(bucketMetrics.bytesTotal); diff --git a/lib/server.js b/lib/server.js index a5bb364b61..e77c7c8c3d 100644 --- a/lib/server.js +++ b/lib/server.js @@ -15,15 +15,11 @@ const { blacklistedPrefixes } = require('../constants'); const api = require('./api/api'); const dataWrapper = require('./data/wrapper'); const kms = require('./kms/wrapper'); -const locationStorageCheck = - require('./api/apiUtils/object/locationStorageCheck'); +const locationStorageCheck = require('./api/apiUtils/object/locationStorageCheck'); const vault = require('./auth/vault'); const metadata = require('./metadata/wrapper'); const { initManagement } = require('./management'); -const { - initManagementClient, - isManagementAgentUsed, -} = require('./management/agentClient'); +const { initManagementClient, isManagementAgentUsed } = require('./management/agentClient'); const { startCleanupJob } = require('./api/apiUtils/rateLimit/cleanup'); const { startRefillJob, stopRefillJob } = require('./api/apiUtils/rateLimit/refillJob'); @@ -46,8 +42,7 @@ updateAllEndpoints(); _config.on('location-constraints-update', () => { if (implName === 'multipleBackends') { const clients = parseLC(_config, vault); - client = new MultipleBackendGateway( - clients, metadata, locationStorageCheck); + client = new MultipleBackendGateway(clients, metadata, locationStorageCheck); } }); @@ -59,8 +54,7 @@ if (_config.localCache) { // stats client const STATS_INTERVAL = 5; // 5 seconds const STATS_EXPIRY = 30; // 30 seconds -const statsClient = new StatsClient(localCacheClient, STATS_INTERVAL, - STATS_EXPIRY); +const statsClient = new StatsClient(localCacheClient, STATS_INTERVAL, STATS_EXPIRY); const enableRemoteManagement = true; class S3Server { @@ -84,7 +78,7 @@ class S3Server { process.on('SIGHUP', this.cleanUp.bind(this)); process.on('SIGQUIT', this.cleanUp.bind(this)); process.on('SIGTERM', this.cleanUp.bind(this)); - process.on('SIGPIPE', () => { }); + process.on('SIGPIPE', () => {}); // This will pick up exceptions up the stack process.on('uncaughtException', err => { // If just send the error object results in empty @@ -130,9 +124,10 @@ class S3Server { const requestStartTime = process.hrtime.bigint(); // Skip server access logs for heartbeat. - const isLoggingEnabled = _config.serverAccessLogs - && (_config.serverAccessLogs.mode === serverAccessLogsModes.LOG_ONLY - || _config.serverAccessLogs.mode === serverAccessLogsModes.ENABLED); + const isLoggingEnabled = + _config.serverAccessLogs && + (_config.serverAccessLogs.mode === serverAccessLogsModes.LOG_ONLY || + _config.serverAccessLogs.mode === serverAccessLogsModes.ENABLED); const isInternalRoute = req.url.startsWith('/_'); const isBackbeatRoute = req.url.startsWith('/_/backbeat/'); if (isLoggingEnabled && (!isInternalRoute || isBackbeatRoute)) { @@ -176,9 +171,7 @@ class S3Server { labels.action = req.apiMethod; } monitoringClient.httpRequestsTotal.labels(labels).inc(); - monitoringClient.httpRequestDurationSeconds - .labels(labels) - .observe(responseTimeInNs / 1e9); + monitoringClient.httpRequestDurationSeconds.labels(labels).observe(responseTimeInNs / 1e9); monitoringClient.httpActiveRequests.dec(); }; res.on('close', monitorEndOfRequest); @@ -231,14 +224,13 @@ class S3Server { }; let reqUids = req.headers['x-scal-request-uids']; - if (reqUids !== undefined && !/*isValidReqUids*/(reqUids.length < 128)) { + if (reqUids !== undefined && !(/*isValidReqUids*/ (reqUids.length < 128))) { // simply ignore invalid id (any user can provide an // invalid request ID through a crafted header) reqUids = undefined; } - const log = (reqUids !== undefined ? - logger.newRequestLoggerFromSerializedUids(reqUids) : - logger.newRequestLogger()); + const log = + reqUids !== undefined ? logger.newRequestLoggerFromSerializedUids(reqUids) : logger.newRequestLogger(); log.end().addDefaultFields(clientInfo); log.debug('received admin request', clientInfo); @@ -292,8 +284,7 @@ class S3Server { server.requestTimeout = 0; // disabling request timeout server.on('connection', socket => { - socket.on('error', err => logger.info('request rejected', - { error: err })); + socket.on('error', err => logger.info('request rejected', { error: err })); }); // https://nodejs.org/dist/latest-v6.x/ @@ -309,8 +300,11 @@ class S3Server { }; const { address } = addr; logger.info('server started', { - address, port, - pid: process.pid, serverIP: address, serverPort: port + address, + port, + pid: process.pid, + serverIP: address, + serverPort: port, }); }); @@ -332,9 +326,9 @@ class S3Server { if (this.config.rateLimiting?.enabled) { stopRefillJob(logger); } - Promise.all(this.servers.map(server => - new Promise(resolve => server.close(resolve)) - )).then(() => process.exit(0)); + Promise.all(this.servers.map(server => new Promise(resolve => server.close(resolve)))).then(() => + process.exit(0), + ); } caughtExceptionShutdown() { @@ -363,10 +357,7 @@ class S3Server { } initiateStartup(log) { - series([ - next => metadata.setup(next), - next => clientCheck(true, log, next), - ], (err, results) => { + series([next => metadata.setup(next), next => clientCheck(true, log, next)], (err, results) => { if (err) { log.warn('initial health check failed, delaying startup', { error: err, @@ -417,8 +408,10 @@ class S3Server { try { logger.info('ServerAccessLogger config', { config: _config.serverAccessLogs }); - if (_config.serverAccessLogs.mode === serverAccessLogsModes.LOG_ONLY - || _config.serverAccessLogs.mode === serverAccessLogsModes.ENABLED) { + if ( + _config.serverAccessLogs.mode === serverAccessLogsModes.LOG_ONLY || + _config.serverAccessLogs.mode === serverAccessLogsModes.ENABLED + ) { var serverAccessLogger = new ServerAccessLogger( _config.serverAccessLogs.outputFile, _config.serverAccessLogs.highWaterMarkBytes, @@ -434,7 +427,6 @@ class S3Server { logger.error('ServerAccessLogger creation error', error); } - this.started = true; }); } @@ -490,8 +482,7 @@ function main() { }); const metricServer = new S3Server(_config); - metricServer.startServer(_config.metricsListenOn, - _config.metricsPort, metricServer.routeAdminRequest); + metricServer.startServer(_config.metricsListenOn, _config.metricsPort, metricServer.routeAdminRequest); } if (_config.isCluster && cluster.isWorker) { const server = new S3Server(_config, cluster.worker); diff --git a/lib/utapi/utapiReindex.js b/lib/utapi/utapiReindex.js index 9bc1522355..ad58f02e81 100644 --- a/lib/utapi/utapiReindex.js +++ b/lib/utapi/utapiReindex.js @@ -4,8 +4,7 @@ const { config } = require('../Config'); const reindexConfig = config.utapi && config.utapi.reindex; if (reindexConfig && reindexConfig.password === undefined) { - reindexConfig.password = config.utapi && config.utapi.redis - && config.utapi.redis.password; + reindexConfig.password = config.utapi && config.utapi.redis && config.utapi.redis.password; } const reindex = new UtapiReindex(reindexConfig); reindex.start(); diff --git a/lib/utapi/utilities.js b/lib/utapi/utilities.js index 32ea8432ac..061b283888 100644 --- a/lib/utapi/utilities.js +++ b/lib/utapi/utilities.js @@ -12,10 +12,13 @@ let utapiConfig; if (utapiVersion === 1 && _config.utapi) { utapiConfig = Object.assign({}, _config.utapi); } else if (utapiVersion === 2) { - utapiConfig = Object.assign({ - tls: _config.https, - suppressedEventFields, - }, _config.utapi || {}); + utapiConfig = Object.assign( + { + tls: _config.https, + suppressedEventFields, + }, + _config.utapi || {}, + ); } const utapi = new UtapiClient(utapiConfig); @@ -33,8 +36,7 @@ const bucketOwnerMetrics = [ function evalAuthInfo(authInfo, canonicalID, action) { let accountId = authInfo.getCanonicalID(); - let userId = authInfo.isRequesterAnIAMUser() ? - authInfo.getShortid() : undefined; + let userId = authInfo.isRequesterAnIAMUser() ? authInfo.getShortid() : undefined; // If action impacts 'numberOfObjectsStored' or 'storageUtilized' metric // only the bucket owner account's metrics should be updated const canonicalIdMatch = authInfo.getCanonicalID() === canonicalID; @@ -48,21 +50,10 @@ function evalAuthInfo(authInfo, canonicalID, action) { }; } -function _listMetrics(host, - port, - metric, - metricType, - timeRange, - accessKey, - secretKey, - verbose, - recent, - ssl) { +function _listMetrics(host, port, metric, metricType, timeRange, accessKey, secretKey, verbose, recent, ssl) { const listAction = recent ? 'ListRecentMetrics' : 'ListMetrics'; // If recent listing, we do not provide `timeRange` in the request - const requestObj = recent - ? { [metric]: metricType } - : { timeRange, [metric]: metricType }; + const requestObj = recent ? { [metric]: metricType } : { timeRange, [metric]: metricType }; const requestBody = JSON.stringify(requestObj); const options = { host, @@ -104,8 +95,7 @@ function _listMetrics(host, }); // TODO: cleanup with refactor of generateV4Headers request.path = `/${metric}`; - auth.client.generateV4Headers(request, { Action: listAction }, - accessKey, secretKey, 's3'); + auth.client.generateV4Headers(request, { Action: listAction }, accessKey, secretKey, 's3'); request.path = `/${metric}?Action=${listAction}`; if (verbose) { logger.info('request headers', { headers: request.getHeaders() }); @@ -132,24 +122,18 @@ function listMetrics(metricType) { // bin/list_bucket_metrics.js when prior method of listing bucket metrics is // no longer supported. if (metricType === 'buckets') { - program - .option('-b, --buckets ', 'Name of bucket(s) with ' + - 'a comma separator if more than one'); + program.option('-b, --buckets ', 'Name of bucket(s) with ' + 'a comma separator if more than one'); } else { program .option('-m, --metric ', 'Metric type') - .option('--buckets ', 'Name of bucket(s) with a comma ' + - 'separator if more than one') - .option('--accounts ', 'Account ID(s) with a comma ' + - 'separator if more than one') - .option('--users ', 'User ID(s) with a comma separator if ' + - 'more than one') + .option('--buckets ', 'Name of bucket(s) with a comma ' + 'separator if more than one') + .option('--accounts ', 'Account ID(s) with a comma ' + 'separator if more than one') + .option('--users ', 'User ID(s) with a comma separator if ' + 'more than one') .option('--service ', 'Name of service'); } program .option('-s, --start ', 'Start of time range') - .option('-r, --recent', 'List metrics including the previous and ' + - 'current 15 minute interval') + .option('-r, --recent', 'List metrics including the previous and ' + 'current 15 minute interval') .option('-e --end ', 'End of time range') .option('-h, --host ', 'Host of the server') .option('-p, --port ', 'Port of the server') @@ -157,7 +141,6 @@ function listMetrics(metricType) { .option('-v, --verbose') .parse(process.argv); - const providedOptions = program.opts(); const { host, port, accessKey, secretKey, start, end, verbose, recent, ssl, metric: metricLvl } = providedOptions; const requiredOptions = { host, port, accessKey, secretKey }; @@ -223,8 +206,7 @@ function listMetrics(metricType) { } } - _listMetrics(host, port, metric, resources, timeRange, accessKey, secretKey, - verbose, recent, ssl); + _listMetrics(host, port, metric, resources, timeRange, accessKey, secretKey, verbose, recent, ssl); } /** @@ -284,9 +266,11 @@ function pushMetric(action, log, metricObj) { let objectDelta = isDelete ? -numberOfObjects : numberOfObjects; // putDeleteMarkerObject does not pass numberOfObjects - if ((action === 'putDeleteMarkerObject' && byteLength === null) - || action === 'replicateDelete' - || action === 'replicateObject') { + if ( + (action === 'putDeleteMarkerObject' && byteLength === null) || + action === 'replicateDelete' || + action === 'replicateObject' + ) { objectDelta = 1; } else if (action === 'multiObjectDelete') { objectDelta = -(numberOfObjects + removedDeleteMarkers); @@ -303,8 +287,10 @@ function pushMetric(action, log, metricObj) { }; // Any operation from lifecycle that does not change object count or size is dropped - const isLifecycle = _config.lifecycleRoleName - && authInfo && authInfo.arn.endsWith(`:assumed-role/${_config.lifecycleRoleName}/backbeat-lifecycle`); + const isLifecycle = + _config.lifecycleRoleName && + authInfo && + authInfo.arn.endsWith(`:assumed-role/${_config.lifecycleRoleName}/backbeat-lifecycle`); if (isLifecycle && !objectDelta && !sizeDelta) { log.trace('ignoring pushMetric from lifecycle service user', { action, bucket, keys }); return undefined; @@ -381,8 +367,7 @@ function getLocationMetric(location, log, cb) { */ function pushLocationMetric(location, byteLength, log, cb) { const locationId = _getLocationId(location); - return utapi.pushLocationMetric(locationId, byteLength, - log.getSerializedUids(), cb); + return utapi.pushLocationMetric(locationId, byteLength, log.getSerializedUids(), cb); } module.exports = { diff --git a/lib/utilities/aclUtils.js b/lib/utilities/aclUtils.js index 7a3b820369..af9563174d 100644 --- a/lib/utilities/aclUtils.js +++ b/lib/utilities/aclUtils.js @@ -4,18 +4,18 @@ const { errors, s3middleware } = require('arsenal'); const constants = require('../../constants'); const escapeForXml = s3middleware.escapeForXml; -const possibleGrantHeaders = ['x-amz-grant-read', 'x-amz-grant-write', - 'x-amz-grant-read-acp', 'x-amz-grant-write-acp', - 'x-amz-grant-full-control']; +const possibleGrantHeaders = [ + 'x-amz-grant-read', + 'x-amz-grant-write', + 'x-amz-grant-read-acp', + 'x-amz-grant-write-acp', + 'x-amz-grant-full-control', +]; const regexpEmailAddress = /^\S+@\S+.\S+$/; const aclUtils = {}; -const grantsByURI = [ - constants.publicId, - constants.allAuthedUsersId, - constants.logId, -]; +const grantsByURI = [constants.publicId, constants.allAuthedUsersId, constants.logId]; /** * handleCannedGrant - Populate grantInfo for a bucketGetACL or objectGetACL @@ -26,86 +26,81 @@ const grantsByURI = [ * are different) * @returns {array} cannedGrants - containing canned ACL settings */ -aclUtils.handleCannedGrant = - function handleCannedGrant(grantType, - ownerGrant, separateBucketOwner) { - const cannedGrants = []; - const actions = { - 'private': () => { - cannedGrants.push(ownerGrant); - }, - 'public-read': () => { - const publicGrant = { - URI: constants.publicId, - permission: 'READ', - }; - cannedGrants.push(ownerGrant, publicGrant); - }, - 'public-read-write': () => { - const publicReadGrant = { - URI: constants.publicId, +aclUtils.handleCannedGrant = function handleCannedGrant(grantType, ownerGrant, separateBucketOwner) { + const cannedGrants = []; + const actions = { + private: () => { + cannedGrants.push(ownerGrant); + }, + 'public-read': () => { + const publicGrant = { + URI: constants.publicId, + permission: 'READ', + }; + cannedGrants.push(ownerGrant, publicGrant); + }, + 'public-read-write': () => { + const publicReadGrant = { + URI: constants.publicId, + permission: 'READ', + }; + const publicWriteGrant = { + URI: constants.publicId, + permission: 'WRITE', + }; + cannedGrants.push(ownerGrant, publicReadGrant, publicWriteGrant); + }, + 'authenticated-read': () => { + const authGrant = { + URI: constants.allAuthedUsersId, + permission: 'READ', + }; + cannedGrants.push(ownerGrant, authGrant); + }, + // Note: log-delivery-write is just for bucketGetACL + 'log-delivery-write': () => { + const logWriteGrant = { + URI: constants.logId, + permission: 'WRITE', + }; + const logReadACPGrant = { + URI: constants.logId, + permission: 'READ_ACP', + }; + cannedGrants.push(ownerGrant, logWriteGrant, logReadACPGrant); + }, + // Note: bucket-owner-read is just for objectGetACL + 'bucket-owner-read': () => { + // If the bucket owner and object owner are different, + // add separate entries for each + if (separateBucketOwner) { + const bucketOwnerReadGrant = { + ID: separateBucketOwner.getOwner(), + displayName: separateBucketOwner.getOwnerDisplayName(), permission: 'READ', }; - const publicWriteGrant = { - URI: constants.publicId, - permission: 'WRITE', - }; - cannedGrants. - push(ownerGrant, publicReadGrant, publicWriteGrant); - }, - 'authenticated-read': () => { - const authGrant = { - URI: constants.allAuthedUsersId, - permission: 'READ', - }; - cannedGrants.push(ownerGrant, authGrant); - }, - // Note: log-delivery-write is just for bucketGetACL - 'log-delivery-write': () => { - const logWriteGrant = { - URI: constants.logId, - permission: 'WRITE', - }; - const logReadACPGrant = { - URI: constants.logId, - permission: 'READ_ACP', + cannedGrants.push(ownerGrant, bucketOwnerReadGrant); + } else { + cannedGrants.push(ownerGrant); + } + }, + // Note: bucket-owner-full-control is just for objectGetACL + 'bucket-owner-full-control': () => { + if (separateBucketOwner) { + const bucketOwnerFCGrant = { + ID: separateBucketOwner.getOwner(), + displayName: separateBucketOwner.getOwnerDisplayName(), + permission: 'FULL_CONTROL', }; - cannedGrants. - push(ownerGrant, logWriteGrant, logReadACPGrant); - }, - // Note: bucket-owner-read is just for objectGetACL - 'bucket-owner-read': () => { - // If the bucket owner and object owner are different, - // add separate entries for each - if (separateBucketOwner) { - const bucketOwnerReadGrant = { - ID: separateBucketOwner.getOwner(), - displayName: separateBucketOwner.getOwnerDisplayName(), - permission: 'READ', - }; - cannedGrants.push(ownerGrant, bucketOwnerReadGrant); - } else { - cannedGrants.push(ownerGrant); - } - }, - // Note: bucket-owner-full-control is just for objectGetACL - 'bucket-owner-full-control': () => { - if (separateBucketOwner) { - const bucketOwnerFCGrant = { - ID: separateBucketOwner.getOwner(), - displayName: separateBucketOwner.getOwnerDisplayName(), - permission: 'FULL_CONTROL', - }; - cannedGrants.push(ownerGrant, bucketOwnerFCGrant); - } else { - cannedGrants.push(ownerGrant); - } - }, - }; - actions[grantType](); - return cannedGrants; + cannedGrants.push(ownerGrant, bucketOwnerFCGrant); + } else { + cannedGrants.push(ownerGrant); + } + }, }; - + actions[grantType](); + return cannedGrants; +}; aclUtils.parseAclXml = function parseAclXml(toBeParsed, log, next) { return parseString(toBeParsed, (err, result) => { @@ -113,24 +108,26 @@ aclUtils.parseAclXml = function parseAclXml(toBeParsed, log, next) { log.debug('invalid xml', { xmlObj: toBeParsed }); return next(errors.MalformedXML); } - if (!result.AccessControlPolicy - || !result.AccessControlPolicy.AccessControlList - || result.AccessControlPolicy.AccessControlList.length !== 1 - || (result.AccessControlPolicy.AccessControlList[0] !== '' && - Object.keys(result.AccessControlPolicy.AccessControlList[0]) - .some(listKey => listKey !== 'Grant'))) { + if ( + !result.AccessControlPolicy || + !result.AccessControlPolicy.AccessControlList || + result.AccessControlPolicy.AccessControlList.length !== 1 || + (result.AccessControlPolicy.AccessControlList[0] !== '' && + Object.keys(result.AccessControlPolicy.AccessControlList[0]).some(listKey => listKey !== 'Grant')) + ) { log.debug('invalid acl', { acl: result }); return next(errors.MalformedACLError); } - const jsonGrants = result - .AccessControlPolicy.AccessControlList[0].Grant; + const jsonGrants = result.AccessControlPolicy.AccessControlList[0].Grant; log.trace('acl grants', { aclGrants: jsonGrants }); - if (!Array.isArray(result.AccessControlPolicy.Owner) - || result.AccessControlPolicy.Owner.length !== 1 - || !Array.isArray(result.AccessControlPolicy.Owner[0].ID) - || result.AccessControlPolicy.Owner[0].ID.length !== 1 - || result.AccessControlPolicy.Owner[0].ID[0] === '') { + if ( + !Array.isArray(result.AccessControlPolicy.Owner) || + result.AccessControlPolicy.Owner.length !== 1 || + !Array.isArray(result.AccessControlPolicy.Owner[0].ID) || + result.AccessControlPolicy.Owner[0].ID.length !== 1 || + result.AccessControlPolicy.Owner[0].ID[0] === '' + ) { return next(errors.MalformedACLError); } const ownerID = result.AccessControlPolicy.Owner[0].ID[0]; @@ -139,8 +136,7 @@ aclUtils.parseAclXml = function parseAclXml(toBeParsed, log, next) { }); }; -aclUtils.getPermissionType = function getPermissionType(identifier, resourceACL, - resourceType) { +aclUtils.getPermissionType = function getPermissionType(identifier, resourceACL, resourceType) { const fullControlIndex = resourceACL.FULL_CONTROL.indexOf(identifier); let writeIndex; if (resourceType === 'bucket') { @@ -196,32 +192,29 @@ aclUtils.isValidCanonicalId = function isValidCanonicalId(canonicalID) { return /^(?=.*?[a-f])(?=.*?[0-9])[a-f0-9]{64}$/.test(canonicalID); }; -aclUtils.reconstructUsersIdentifiedByEmail = - function reconstruct(userInfofromVault, userGrantInfo) { - return userGrantInfo.map(item => { - const userEmail = item.identifier.toLowerCase(); - const user = {}; - // Find the full user grant info based on email - const userId = userInfofromVault - .find(elem => elem.email.toLowerCase() === userEmail); - // Set the identifier to be the canonicalID instead of email - user.identifier = userId.canonicalID; - user.userIDType = 'id'; - // copy over ACL grant type: i.e. READ/WRITE... - user.grantType = item.grantType; - return user; - }); - }; +aclUtils.reconstructUsersIdentifiedByEmail = function reconstruct(userInfofromVault, userGrantInfo) { + return userGrantInfo.map(item => { + const userEmail = item.identifier.toLowerCase(); + const user = {}; + // Find the full user grant info based on email + const userId = userInfofromVault.find(elem => elem.email.toLowerCase() === userEmail); + // Set the identifier to be the canonicalID instead of email + user.identifier = userId.canonicalID; + user.userIDType = 'id'; + // copy over ACL grant type: i.e. READ/WRITE... + user.grantType = item.grantType; + return user; + }); +}; -aclUtils.sortHeaderGrants = - function sortHeaderGrants(allGrantHeaders, addACLParams) { - allGrantHeaders.forEach(item => { - if (item) { - addACLParams[item.grantType].push(item.identifier); - } - }); - return addACLParams; - }; +aclUtils.sortHeaderGrants = function sortHeaderGrants(allGrantHeaders, addACLParams) { + allGrantHeaders.forEach(item => { + if (item) { + addACLParams[item.grantType].push(item.identifier); + } + }); + return addACLParams; +}; /** * convertToXml - Converts the `grantInfo` object (defined in `objectGetACL()`) @@ -234,14 +227,14 @@ aclUtils.convertToXml = grantInfo => { const { grants, ownerInfo } = grantInfo; const xml = []; - xml.push('', + xml.push( + '', '', '', `${ownerInfo.ID}`, - `${escapeForXml(ownerInfo.displayName)}` + - '', + `${escapeForXml(ownerInfo.displayName)}` + '', '', - '' + '', ); grants.forEach(grant => { @@ -250,32 +243,25 @@ aclUtils.convertToXml = grantInfo => { // The `` tag has different attributes depending on whether the // grant has an ID or URI if (grant.ID) { - xml.push('', - `${grant.ID}` + xml.push( + '', + `${grant.ID}`, ); } else if (grant.URI) { - xml.push('', - `${escapeForXml(grant.URI)}` + xml.push( + '', + `${escapeForXml(grant.URI)}`, ); } if (grant.displayName) { - xml.push(`${escapeForXml(grant.displayName)}` + - '' - ); + xml.push(`${escapeForXml(grant.displayName)}` + ''); } - xml.push('', - `${grant.permission}`, - '' - ); + xml.push('', `${grant.permission}`, ''); }); - xml.push('', - '' - ); + xml.push('', ''); return xml.join(''); }; @@ -303,9 +289,11 @@ aclUtils.checkGrantHeaderValidity = function checkGrantHeaderValidity(headers) { const identifier = singleGrantArr[0].trim().toLowerCase(); const value = singleGrantArr[1].trim(); if (identifier === 'uri') { - if (value !== constants.publicId && + if ( + value !== constants.publicId && value !== constants.allAuthedUsersId && - value !== constants.logId) { + value !== constants.logId + ) { return false; } } else if (identifier === 'emailaddress') { @@ -346,13 +334,7 @@ function getGrants(acl) { * @returns {array} canonicalIDs - array of unique canonicalIDs from acl */ aclUtils.getCanonicalIDs = function getCanonicalIDs(acl) { - const aclGrantees = [].concat( - acl.FULL_CONTROL, - acl.WRITE, - acl.WRITE_ACP, - acl.READ, - acl.READ_ACP - ); + const aclGrantees = [].concat(acl.FULL_CONTROL, acl.WRITE, acl.WRITE_ACP, acl.READ, acl.READ_ACP); const uniqueGrantees = Array.from(new Set(aclGrantees)); // grantees can be a mix of canonicalIDs and predefined groups in the form // of uri, so filter out only canonicalIDs @@ -367,11 +349,12 @@ aclUtils.getCanonicalIDs = function getCanonicalIDs(acl) { aclUtils.getUriGrantInfo = function getUriGrantInfo(acl) { const grants = getGrants(acl); const uriGrantInfo = []; - const validGrants = Object.entries(grants) - .filter(([permission, grantees]) => permission - && Array.isArray(grantees)); + const validGrants = Object.entries(grants).filter( + ([permission, grantees]) => permission && Array.isArray(grantees), + ); validGrants.forEach(([permission, grantees]) => { - grantees.filter(grantee => grantsByURI.includes(grantee)) + grantees + .filter(grantee => grantsByURI.includes(grantee)) .forEach(grantee => { uriGrantInfo.push({ URI: grantee, @@ -391,16 +374,15 @@ aclUtils.getUriGrantInfo = function getUriGrantInfo(acl) { * @returns {array} individualGrantInfo - array of grants mapped to * canonicalID/email */ -aclUtils.getIndividualGrants = function getIndividualGrants(acl, canonicalIDs, - emails) { +aclUtils.getIndividualGrants = function getIndividualGrants(acl, canonicalIDs, emails) { const grants = getGrants(acl); const individualGrantInfo = []; - const validGrants = Object.entries(grants) - .filter(([permission, grantees]) => permission - && Array.isArray(grantees)); + const validGrants = Object.entries(grants).filter( + ([permission, grantees]) => permission && Array.isArray(grantees), + ); validGrants.forEach(([permission, grantees]) => { - grantees.filter(grantee => canonicalIDs.includes(grantee) - && emails[grantee]) + grantees + .filter(grantee => canonicalIDs.includes(grantee) && emails[grantee]) .forEach(grantee => { individualGrantInfo.push({ ID: grantee, diff --git a/lib/utilities/collectCorsHeaders.js b/lib/utilities/collectCorsHeaders.js index 286e510953..a2e655c0db 100644 --- a/lib/utilities/collectCorsHeaders.js +++ b/lib/utilities/collectCorsHeaders.js @@ -1,5 +1,4 @@ -const { findCorsRule, generateCorsResHeaders } = - require('../api/apiUtils/object/corsResponse.js'); +const { findCorsRule, generateCorsResHeaders } = require('../api/apiUtils/object/corsResponse.js'); /** * collectCorsHeaders - gather any relevant CORS headers diff --git a/lib/utilities/collectResponseHeaders.js b/lib/utilities/collectResponseHeaders.js index a754300c62..a8136cfe25 100644 --- a/lib/utilities/collectResponseHeaders.js +++ b/lib/utilities/collectResponseHeaders.js @@ -1,6 +1,5 @@ const { getVersionIdResHeader } = require('../api/apiUtils/object/versioning'); -const checkUserMetadataSize - = require('../api/apiUtils/object/checkUserMetadataSize'); +const checkUserMetadataSize = require('../api/apiUtils/object/checkUserMetadataSize'); const { getAmzRestoreResHeader } = require('../api/apiUtils/object/coldStorage'); const { config } = require('../Config'); const { getKeyIdFromArn } = require('arsenal/build/lib/network/KMSInterface'); @@ -16,38 +15,36 @@ const { getKeyIdFromArn } = require('arsenal/build/lib/network/KMSInterface'); * @return {object} responseMetaHeaders headers with object metadata to include * in response to client */ -function collectResponseHeaders(objectMD, corsHeaders, versioningCfg, - returnTagCount) { +function collectResponseHeaders(objectMD, corsHeaders, versioningCfg, returnTagCount) { // Add user meta headers from objectMD let responseMetaHeaders = Object.assign({}, corsHeaders); - Object.keys(objectMD).filter(val => (val.startsWith('x-amz-meta-'))) - .forEach(id => { responseMetaHeaders[id] = objectMD[id]; }); + Object.keys(objectMD) + .filter(val => val.startsWith('x-amz-meta-')) + .forEach(id => { + responseMetaHeaders[id] = objectMD[id]; + }); // Check user metadata size responseMetaHeaders = checkUserMetadataSize(responseMetaHeaders); // TODO: When implement lifecycle, add additional response headers // http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectHEAD.html - responseMetaHeaders['x-amz-version-id'] = - getVersionIdResHeader(versioningCfg, objectMD); + responseMetaHeaders['x-amz-version-id'] = getVersionIdResHeader(versioningCfg, objectMD); if (objectMD['x-amz-website-redirect-location']) { - responseMetaHeaders['x-amz-website-redirect-location'] = - objectMD['x-amz-website-redirect-location']; + responseMetaHeaders['x-amz-website-redirect-location'] = objectMD['x-amz-website-redirect-location']; } if (objectMD['x-amz-storage-class'] !== 'STANDARD') { - responseMetaHeaders['x-amz-storage-class'] = - objectMD['x-amz-storage-class']; + responseMetaHeaders['x-amz-storage-class'] = objectMD['x-amz-storage-class']; } if (objectMD['x-amz-server-side-encryption']) { - responseMetaHeaders['x-amz-server-side-encryption'] - = objectMD['x-amz-server-side-encryption']; + responseMetaHeaders['x-amz-server-side-encryption'] = objectMD['x-amz-server-side-encryption']; } const kmsKey = objectMD['x-amz-server-side-encryption-aws-kms-key-id']; - if (kmsKey && - objectMD['x-amz-server-side-encryption'] === 'aws:kms') { - responseMetaHeaders['x-amz-server-side-encryption-aws-kms-key-id'] - = config.kmsHideScalityArn ? getKeyIdFromArn(kmsKey) : kmsKey; + if (kmsKey && objectMD['x-amz-server-side-encryption'] === 'aws:kms') { + responseMetaHeaders['x-amz-server-side-encryption-aws-kms-key-id'] = config.kmsHideScalityArn + ? getKeyIdFromArn(kmsKey) + : kmsKey; } const restoreHeader = getAmzRestoreResHeader(objectMD); @@ -65,8 +62,7 @@ function collectResponseHeaders(objectMD, corsHeaders, versioningCfg, responseMetaHeaders['Cache-Control'] = objectMD['cache-control']; } if (objectMD['content-disposition']) { - responseMetaHeaders['Content-Disposition'] - = objectMD['content-disposition']; + responseMetaHeaders['Content-Disposition'] = objectMD['content-disposition']; } if (objectMD['content-encoding']) { responseMetaHeaders['Content-Encoding'] = objectMD['content-encoding']; @@ -78,40 +74,30 @@ function collectResponseHeaders(objectMD, corsHeaders, versioningCfg, // Note: ETag must have a capital "E" and capital "T" for cosbench // to work. responseMetaHeaders.ETag = `"${objectMD['content-md5']}"`; - responseMetaHeaders['Last-Modified'] = - new Date(objectMD['last-modified']).toUTCString(); + responseMetaHeaders['Last-Modified'] = new Date(objectMD['last-modified']).toUTCString(); if (objectMD['content-type']) { responseMetaHeaders['Content-Type'] = objectMD['content-type']; } - if (returnTagCount && objectMD.tags && - Object.keys(objectMD.tags).length > 0) { - responseMetaHeaders['x-amz-tagging-count'] = - Object.keys(objectMD.tags).length; + if (returnTagCount && objectMD.tags && Object.keys(objectMD.tags).length > 0) { + responseMetaHeaders['x-amz-tagging-count'] = Object.keys(objectMD.tags).length; } - const hasRetentionInfo = objectMD.retentionMode - && objectMD.retentionDate; + const hasRetentionInfo = objectMD.retentionMode && objectMD.retentionDate; if (hasRetentionInfo) { - responseMetaHeaders['x-amz-object-lock-retain-until-date'] - = objectMD.retentionDate; - responseMetaHeaders['x-amz-object-lock-mode'] - = objectMD.retentionMode; + responseMetaHeaders['x-amz-object-lock-retain-until-date'] = objectMD.retentionDate; + responseMetaHeaders['x-amz-object-lock-mode'] = objectMD.retentionMode; } if (objectMD.legalHold !== undefined) { - responseMetaHeaders['x-amz-object-lock-legal-hold'] - = objectMD.legalHold ? 'ON' : 'OFF'; + responseMetaHeaders['x-amz-object-lock-legal-hold'] = objectMD.legalHold ? 'ON' : 'OFF'; } if (objectMD.replicationInfo && objectMD.replicationInfo.status) { - responseMetaHeaders['x-amz-replication-status'] = - objectMD.replicationInfo.status; + responseMetaHeaders['x-amz-replication-status'] = objectMD.replicationInfo.status; } if (Array.isArray(objectMD?.replicationInfo?.backends)) { objectMD.replicationInfo.backends.forEach(backend => { const { status, site, dataStoreVersionId } = backend; - responseMetaHeaders[`x-amz-meta-${site}-replication-status`] = - status; + responseMetaHeaders[`x-amz-meta-${site}-replication-status`] = status; if (status === 'COMPLETED' && dataStoreVersionId) { - responseMetaHeaders[`x-amz-meta-${site}-version-id`] = - dataStoreVersionId; + responseMetaHeaders[`x-amz-meta-${site}-version-id`] = dataStoreVersionId; } }); } diff --git a/lib/utilities/healthcheckHandler.js b/lib/utilities/healthcheckHandler.js index 35186299f8..58e29d0257 100644 --- a/lib/utilities/healthcheckHandler.js +++ b/lib/utilities/healthcheckHandler.js @@ -41,12 +41,7 @@ function writeResponse(res, error, log, results, cb) { * @return {undefined} */ function clientCheck(flightCheckOnStartUp, log, cb) { - const clients = [ - data, - metadata, - vault, - kms, - ]; + const clients = [data, metadata, vault, kms]; const clientTasks = []; clients.forEach(client => { if (typeof client.checkHealth === 'function') { @@ -65,21 +60,27 @@ function clientCheck(flightCheckOnStartUp, log, cb) { // Each result in the array represents one client (data, metadata, vault, kms) // with its backend locations as keys. // If ALL backend/locations of ANY client have errors, the overall check fails. - const { obj, fail } = results.reduce((acc, clientResult) => { - Object.assign(acc.obj, clientResult); + const { obj, fail } = results.reduce( + (acc, clientResult) => { + Object.assign(acc.obj, clientResult); - // Check if ALL backends/locations of this client have errors - const keys = Object.keys(clientResult); - // eslint-disable-next-line no-param-reassign - acc.fail ||= keys.length > 0 && keys.every(k => - // if there is an error from an external backend, - // only return a 500 if it is on startup - // (flightCheckOnStartUp set to true) - clientResult[k].error && (flightCheckOnStartUp || !clientResult[k].external) - ); + // Check if ALL backends/locations of this client have errors + const keys = Object.keys(clientResult); + // eslint-disable-next-line no-param-reassign + acc.fail ||= + keys.length > 0 && + keys.every( + k => + // if there is an error from an external backend, + // only return a 500 if it is on startup + // (flightCheckOnStartUp set to true) + clientResult[k].error && (flightCheckOnStartUp || !clientResult[k].external), + ); - return acc; - }, { obj: {}, fail: false }); + return acc; + }, + { obj: {}, fail: false }, + ); if (fail) { return cb(errors.InternalError, obj); } @@ -106,8 +107,7 @@ function routeHandler(deep, req, res, log, statsClient, cb) { } function checkIP(clientIP) { - return ipCheck.ipMatchCidrList( - _config.healthChecks.allowFrom, clientIP); + return ipCheck.ipMatchCidrList(_config.healthChecks.allowFrom, clientIP); } /** @@ -139,8 +139,7 @@ function healthcheckHandler(clientIP, req, res, log, statsClient, deep) { if (!checkIP(clientIP)) { return healthcheckEndHandler(errors.AccessDenied, []); } - return routeHandler(deep, req, res, log, statsClient, - healthcheckEndHandler); + return routeHandler(deep, req, res, log, statsClient, healthcheckEndHandler); } module.exports = { diff --git a/lib/utilities/internalHandlers.js b/lib/utilities/internalHandlers.js index d3379158c0..c52d790489 100644 --- a/lib/utilities/internalHandlers.js +++ b/lib/utilities/internalHandlers.js @@ -1,7 +1,6 @@ const { routeBackbeat } = require('../routes/routeBackbeat'); const routeMetadata = require('../routes/routeMetadata'); -const routeWorkflowEngineOperator = - require('../routes/routeWorkflowEngineOperator'); +const routeWorkflowEngineOperator = require('../routes/routeWorkflowEngineOperator'); const { reportHandler } = require('./reportHandler'); const routeVeeam = require('../routes/routeVeeam').routeVeeam; const { healthcheckHandler } = require('./healthcheckHandler'); diff --git a/lib/utilities/legacyAWSBehavior.js b/lib/utilities/legacyAWSBehavior.js index c8a457a687..859d064972 100644 --- a/lib/utilities/legacyAWSBehavior.js +++ b/lib/utilities/legacyAWSBehavior.js @@ -10,8 +10,10 @@ const { config } = require('../Config'); */ function isLegacyAwsBehavior(locationConstraint) { - return (config.locationConstraints[locationConstraint] && - config.locationConstraints[locationConstraint].legacyAwsBehavior); + return ( + config.locationConstraints[locationConstraint] && + config.locationConstraints[locationConstraint].legacyAwsBehavior + ); } module.exports = isLegacyAwsBehavior; diff --git a/lib/utilities/monitoringHandler.js b/lib/utilities/monitoringHandler.js index 712a083067..d6b79633da 100644 --- a/lib/utilities/monitoringHandler.js +++ b/lib/utilities/monitoringHandler.js @@ -114,69 +114,72 @@ if (config.isQuotaEnabled) { // labels and buckets. const lifecycleDuration = new client.Histogram({ name: 's3_lifecycle_duration_seconds', - help: 'Duration of the lifecycle operation, calculated from the theoretical date to the end ' + - 'of the operation', + help: 'Duration of the lifecycle operation, calculated from the theoretical date to the end ' + 'of the operation', labelNames: ['type', 'location'], buckets: [0.2, 1, 5, 30, 120, 600, 3600, 4 * 3600, 8 * 3600, 16 * 3600, 24 * 3600], }); -function promMetrics(method, bucketName, code, action, - newByteLength, oldByteLength, isVersionedObj, - numOfObjectsRemoved, ingestSize) { +function promMetrics( + method, + bucketName, + code, + action, + newByteLength, + oldByteLength, + isVersionedObj, + numOfObjectsRemoved, + ingestSize, +) { let bytes; switch (action) { - case 'putObject': - case 'copyObject': - case 'putObjectPart': - if (code === '200') { - bytes = newByteLength - (isVersionedObj ? 0 : oldByteLength); - httpRequestSizeBytes - .labels(method, action, code) - .observe(newByteLength); - dataDiskAvailable.dec(bytes); - dataDiskFree.dec(bytes); - if (ingestSize) { - numberOfIngestedObjects.inc(); - dataIngested.inc(ingestSize); + case 'putObject': + case 'copyObject': + case 'putObjectPart': + if (code === '200') { + bytes = newByteLength - (isVersionedObj ? 0 : oldByteLength); + httpRequestSizeBytes.labels(method, action, code).observe(newByteLength); + dataDiskAvailable.dec(bytes); + dataDiskFree.dec(bytes); + if (ingestSize) { + numberOfIngestedObjects.inc(); + dataIngested.inc(ingestSize); + } + numberOfObjects.inc(); } - numberOfObjects.inc(); - } - break; - case 'createBucket': - if (code === '200') { - numberOfBuckets.inc(); - } - break; - case 'getObject': - if (code === '200') { - httpResponseSizeBytes - .labels(method, action, code) - .observe(newByteLength); - } - break; - case 'deleteBucket': - case 'deleteBucketWebsite': - if (code === '200' || code === '204') { - numberOfBuckets.dec(); - } - break; - case 'deleteObject': - case 'abortMultipartUpload': - case 'multiObjectDelete': - if (code === '200') { - dataDiskAvailable.inc(newByteLength); - dataDiskFree.inc(newByteLength); - const objs = numOfObjectsRemoved || 1; - numberOfObjects.dec(objs); - if (ingestSize) { - numberOfIngestedObjects.dec(objs); - dataIngested.dec(ingestSize); + break; + case 'createBucket': + if (code === '200') { + numberOfBuckets.inc(); } - } - break; - default: - break; + break; + case 'getObject': + if (code === '200') { + httpResponseSizeBytes.labels(method, action, code).observe(newByteLength); + } + break; + case 'deleteBucket': + case 'deleteBucketWebsite': + if (code === '200' || code === '204') { + numberOfBuckets.dec(); + } + break; + case 'deleteObject': + case 'abortMultipartUpload': + case 'multiObjectDelete': + if (code === '200') { + dataDiskAvailable.inc(newByteLength); + dataDiskFree.inc(newByteLength); + const objs = numOfObjectsRemoved || 1; + numberOfObjects.dec(objs); + if (ingestSize) { + numberOfIngestedObjects.dec(objs); + dataIngested.dec(ingestSize); + } + } + break; + default: + break; } } @@ -216,8 +219,7 @@ function writeResponse(res, error, results, cb) { } const registry = config.isCluster ? new client.AggregatorRegistry() : client.register; -const getMetrics = config.isCluster ? - registry.clusterMetrics.bind(registry) : registry.metrics.bind(registry); +const getMetrics = config.isCluster ? registry.clusterMetrics.bind(registry) : registry.metrics.bind(registry); async function routeHandler(req, res, cb) { if (req.method !== 'GET') { diff --git a/lib/utilities/reportHandler.js b/lib/utilities/reportHandler.js index c468d20c4b..f886916302 100644 --- a/lib/utilities/reportHandler.js +++ b/lib/utilities/reportHandler.js @@ -57,7 +57,7 @@ function getCapabilities(cfg = config) { // Consistency & safety checks for capabilities that depend on other config values const localVolumeCap = process.env.LOCAL_VOLUME_CAPABILITY || 'true'; - caps.locationTypeLocal &&= (localVolumeCap === '1' || localVolumeCap.toLowerCase() === 'true'); + caps.locationTypeLocal &&= localVolumeCap === '1' || localVolumeCap.toLowerCase() === 'true'; caps.secureChannelOptimizedPath &&= hasWSOptionalDependencies(); caps.managedLifecycle &&= cfg.supportedLifecycleRules.includes('Expiration'); caps.managedLifecycleTransition &&= cfg.supportedLifecycleRules.includes('Transition'); @@ -87,8 +87,10 @@ function cleanup(obj) { } function isAuthorized(clientIP, req) { - return ipCheck.ipMatchCidrList(config.healthChecks.allowFrom, clientIP) && - req.headers['x-scal-report-token'] === config.reportToken; + return ( + ipCheck.ipMatchCidrList(config.healthChecks.allowFrom, clientIP) && + req.headers['x-scal-report-token'] === config.reportToken + ); } function getGitVersion(cb) { @@ -107,22 +109,28 @@ function getSystemStats() { const cpuInfo = os.cpus(); const model = cpuInfo[0].model; const speed = cpuInfo[0].speed; - const times = cpuInfo. - map(c => c.times). - reduce((prev, cur) => - Object.assign({}, { - user: prev.user + cur.user, - nice: prev.nice + cur.nice, - sys: prev.sys + cur.sys, - idle: prev.idle + cur.idle, - irq: prev.irq + cur.irq, - }), { + const times = cpuInfo + .map(c => c.times) + .reduce( + (prev, cur) => + Object.assign( + {}, + { + user: prev.user + cur.user, + nice: prev.nice + cur.nice, + sys: prev.sys + cur.sys, + idle: prev.idle + cur.idle, + irq: prev.irq + cur.irq, + }, + ), + { user: 0, nice: 0, sys: 0, idle: 0, irq: 0, - }); + }, + ); return { memory: { @@ -252,18 +260,19 @@ function _getMetricsByLocation(endpoint, sites, requestMethod, log, cb) { async.mapLimit( sites, ASYNCLIMIT, - (site, next) => requestMethod(endpoint, site, log, (err, res) => { - if (err) { - log.debug('Error in retrieving site metrics', { - method: '_getMetricsByLocation', - error: err, - site, - requestType: requestMethod.name, - }); - return next(null, { site, stats: {} }); - } - return next(null, { site, stats: res }); - }), + (site, next) => + requestMethod(endpoint, site, log, (err, res) => { + if (err) { + log.debug('Error in retrieving site metrics', { + method: '_getMetricsByLocation', + error: err, + site, + requestType: requestMethod.name, + }); + return next(null, { site, stats: {} }); + } + return next(null, { site, stats: res }); + }), (err, locStats) => { if (err) { log.error('failed to get stats for site', { @@ -278,7 +287,7 @@ function _getMetricsByLocation(endpoint, sites, requestMethod, log, cb) { retObj[locStat.site] = locStat.stats; }); return cb(null, retObj); - } + }, ); } @@ -286,19 +295,21 @@ function _getMetrics(sites, requestMethod, log, cb, _testConfig) { const conf = (_testConfig && _testConfig.backbeat) || config.backbeat; const { host, port } = conf; const endpoint = `http://${host}:${port}`; - return async.parallel({ - all: done => requestMethod(endpoint, 'all', log, done), - byLocation: done => _getMetricsByLocation(endpoint, sites, - requestMethod, log, done), - }, (err, res) => { - if (err) { - return cb(err); - } - const all = (res && res.all) || {}; - const byLocation = (res && res.byLocation) || {}; - const retObj = Object.assign({}, all, { byLocation }); - return cb(null, retObj); - }); + return async.parallel( + { + all: done => requestMethod(endpoint, 'all', log, done), + byLocation: done => _getMetricsByLocation(endpoint, sites, requestMethod, log, done), + }, + (err, res) => { + if (err) { + return cb(err); + } + const all = (res && res.all) || {}; + const byLocation = (res && res.byLocation) || {}; + const retObj = Object.assign({}, all, { byLocation }); + return cb(null, retObj); + }, + ); } function getCRRMetrics(log, cb, _testConfig) { @@ -307,58 +318,73 @@ function getCRRMetrics(log, cb, _testConfig) { }); const { replicationEndpoints } = _testConfig || config; const sites = replicationEndpoints.map(endpoint => endpoint.site); - return _getMetrics(sites, _crrMetricRequest, log, (err, retObj) => { - if (err) { - log.error('failed to get CRR stats', { - method: 'getCRRMetrics', - error: err, - }); - return cb(null, {}); - } - return cb(null, retObj); - }, _testConfig); + return _getMetrics( + sites, + _crrMetricRequest, + log, + (err, retObj) => { + if (err) { + log.error('failed to get CRR stats', { + method: 'getCRRMetrics', + error: err, + }); + return cb(null, {}); + } + return cb(null, retObj); + }, + _testConfig, + ); } function getIngestionMetrics(sites, log, cb, _testConfig) { log.debug('request Ingestion metrics from backbeat api', { method: 'getIngestionMetrics', }); - return _getMetrics(sites, _ingestionMetricRequest, log, (err, retObj) => { - if (err) { - log.error('failed to get Ingestion stats', { - method: 'getIngestionMetrics', - error: err, - }); - return cb(null, {}); - } - return cb(null, retObj); - }, _testConfig); + return _getMetrics( + sites, + _ingestionMetricRequest, + log, + (err, retObj) => { + if (err) { + log.error('failed to get Ingestion stats', { + method: 'getIngestionMetrics', + error: err, + }); + return cb(null, {}); + } + return cb(null, retObj); + }, + _testConfig, + ); } function _getStates(statusPath, schedulePath, log, cb, _testConfig) { const conf = (_testConfig && _testConfig.backbeat) || config.backbeat; const { host, port } = conf; const endpoint = `http://${host}:${port}`; - async.parallel({ - states: done => _makeRequest(endpoint, statusPath, done), - schedules: done => _makeRequest(endpoint, schedulePath, done), - }, (err, res) => { - if (err) { - return cb(err); - } - const locationSchedules = {}; - Object.keys(res.schedules).forEach(loc => { - const val = res.schedules[loc]; - if (!isNaN(Date.parse(val))) { - locationSchedules[loc] = new Date(val); + async.parallel( + { + states: done => _makeRequest(endpoint, statusPath, done), + schedules: done => _makeRequest(endpoint, schedulePath, done), + }, + (err, res) => { + if (err) { + return cb(err); } - }); - const retObj = { - states: res.states || {}, - schedules: locationSchedules, - }; - return cb(null, retObj); - }); + const locationSchedules = {}; + Object.keys(res.schedules).forEach(loc => { + const val = res.schedules[loc]; + if (!isNaN(Date.parse(val))) { + locationSchedules[loc] = new Date(val); + } + }); + const retObj = { + states: res.states || {}, + schedules: locationSchedules, + }; + return cb(null, retObj); + }, + ); } function getReplicationStates(log, cb, _testConfig) { @@ -366,25 +392,31 @@ function getReplicationStates(log, cb, _testConfig) { method: 'getReplicationStates', }); const { crrStatus, crrSchedules } = REQ_PATHS; - return _getStates(crrStatus, crrSchedules, log, (err, res) => { - if (err) { - if (err === 'responseError') { - log.error('error response from backbeat api', { - error: res, - method: 'getReplicationStates', - service: 'replication', - }); - } else { - log.error('unable to perform request to backbeat api', { - error: err, - method: 'getReplicationStates', - service: 'replication', - }); + return _getStates( + crrStatus, + crrSchedules, + log, + (err, res) => { + if (err) { + if (err === 'responseError') { + log.error('error response from backbeat api', { + error: res, + method: 'getReplicationStates', + service: 'replication', + }); + } else { + log.error('unable to perform request to backbeat api', { + error: err, + method: 'getReplicationStates', + service: 'replication', + }); + } + return cb(null, {}); } - return cb(null, {}); - } - return cb(null, res); - }, _testConfig); + return cb(null, res); + }, + _testConfig, + ); } function getIngestionStates(log, cb, _testConfig) { @@ -392,67 +424,81 @@ function getIngestionStates(log, cb, _testConfig) { method: 'getIngestionStates', }); const { ingestionStatus, ingestionSchedules } = REQ_PATHS; - return _getStates(ingestionStatus, ingestionSchedules, log, (err, res) => { - if (err) { - if (err === 'responseError') { - log.error('error response from backbeat api', { - error: res, - method: 'getIngestionStates', - service: 'ingestion', - }); - } else { - log.error('unable to perform request to backbeat api', { - error: err, - method: 'getIngestionStates', - service: 'ingestion', - }); + return _getStates( + ingestionStatus, + ingestionSchedules, + log, + (err, res) => { + if (err) { + if (err === 'responseError') { + log.error('error response from backbeat api', { + error: res, + method: 'getIngestionStates', + service: 'ingestion', + }); + } else { + log.error('unable to perform request to backbeat api', { + error: err, + method: 'getIngestionStates', + service: 'ingestion', + }); + } + return cb(null, {}); } - return cb(null, {}); - } - return cb(null, res); - }, _testConfig); + return cb(null, res); + }, + _testConfig, + ); } function getIngestionInfo(log, cb, _testConfig) { log.debug('requesting location ingestion info from backbeat api', { method: 'getIngestionInfo', }); - async.waterfall([ - done => getIngestionStates(log, done, _testConfig), - (stateObj, done) => { - // if getIngestionStates returned an error or the returned object - // did not return an expected response - if (Object.keys(stateObj).length === 0 || !stateObj.states) { - log.debug('no ingestion locations found', { - method: 'getIngestionInfo', - }); - return done(null, stateObj, {}); - } - const sites = Object.keys(stateObj.states); - return getIngestionMetrics(sites, log, (err, res) => { - if (err) { - log.error('failed to get Ingestion stats', { + async.waterfall( + [ + done => getIngestionStates(log, done, _testConfig), + (stateObj, done) => { + // if getIngestionStates returned an error or the returned object + // did not return an expected response + if (Object.keys(stateObj).length === 0 || !stateObj.states) { + log.debug('no ingestion locations found', { method: 'getIngestionInfo', - error: err, }); return done(null, stateObj, {}); } - return done(null, stateObj, res); - }, _testConfig); - }, - ], (err, stateObj, metricObj) => { - if (err) { - log.error('failed to get ingestion info', { - method: 'getIngestionInfo', - error: err, + const sites = Object.keys(stateObj.states); + return getIngestionMetrics( + sites, + log, + (err, res) => { + if (err) { + log.error('failed to get Ingestion stats', { + method: 'getIngestionInfo', + error: err, + }); + return done(null, stateObj, {}); + } + return done(null, stateObj, res); + }, + _testConfig, + ); + }, + ], + (err, stateObj, metricObj) => { + if (err) { + log.error('failed to get ingestion info', { + method: 'getIngestionInfo', + error: err, + }); + return cb(null, {}); + } + return cb(null, { + metrics: metricObj, + status: stateObj, }); - return cb(null, {}); - } - return cb(null, { - metrics: metricObj, - status: stateObj, - }); - }); + }, + ); } /** @@ -478,52 +524,54 @@ function reportHandler(clientIP, req, res, log) { } // TODO propagate value of req.headers['x-scal-report-skip-cache'] - async.parallel({ - getUUID: cb => metadata.getUUID(log, cb), - getMDDiskUsage: cb => metadata.getDiskUsage(log, cb), - getDataDiskUsage: cb => data.getDiskUsage(log, cb), - getVersion: cb => getGitVersion(cb), - getObjectCount: cb => metadata.countItems(log, cb), - getCRRMetrics: cb => getCRRMetrics(log, cb), - getReplicationStates: cb => getReplicationStates(log, cb), - getIngestionInfo: cb => getIngestionInfo(log, cb), - getVaultReport: cb => vault.report(log, cb), - }, - (err, results) => { - if (err) { - res.writeHead(500, { 'Content-Type': 'application/json' }); - res.write(JSON.stringify(err)); - log.errorEnd('could not gather report', { error: err }); - } else { - const getObjectCount = results.getObjectCount; - const crrStatsObj = Object.assign({}, results.getCRRMetrics); - crrStatsObj.stalled = { count: getObjectCount.stalled || 0 }; - delete getObjectCount.stalled; - const response = { - utcTime: new Date(), - uuid: results.getUUID, - reportModelVersion: REPORT_MODEL_VERSION, - - mdDiskUsage: results.getMDDiskUsage, - dataDiskUsage: results.getDataDiskUsage, - serverVersion: results.getVersion, - systemStats: getSystemStats(), - itemCounts: getObjectCount, - crrStats: crrStatsObj, - repStatus: results.getReplicationStates, - config: cleanup(config), - capabilities: getCapabilities(), - ingestStats: results.getIngestionInfo.metrics, - ingestStatus: results.getIngestionInfo.status, - vaultReport: results.getVaultReport, - }; - monitoring.crrCacheToProm(results); - res.writeHead(200, { 'Content-Type': 'application/json' }); - res.write(JSON.stringify(response)); - log.end().debug('report handler finished'); - } - res.end(); - }); + async.parallel( + { + getUUID: cb => metadata.getUUID(log, cb), + getMDDiskUsage: cb => metadata.getDiskUsage(log, cb), + getDataDiskUsage: cb => data.getDiskUsage(log, cb), + getVersion: cb => getGitVersion(cb), + getObjectCount: cb => metadata.countItems(log, cb), + getCRRMetrics: cb => getCRRMetrics(log, cb), + getReplicationStates: cb => getReplicationStates(log, cb), + getIngestionInfo: cb => getIngestionInfo(log, cb), + getVaultReport: cb => vault.report(log, cb), + }, + (err, results) => { + if (err) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.write(JSON.stringify(err)); + log.errorEnd('could not gather report', { error: err }); + } else { + const getObjectCount = results.getObjectCount; + const crrStatsObj = Object.assign({}, results.getCRRMetrics); + crrStatsObj.stalled = { count: getObjectCount.stalled || 0 }; + delete getObjectCount.stalled; + const response = { + utcTime: new Date(), + uuid: results.getUUID, + reportModelVersion: REPORT_MODEL_VERSION, + + mdDiskUsage: results.getMDDiskUsage, + dataDiskUsage: results.getDataDiskUsage, + serverVersion: results.getVersion, + systemStats: getSystemStats(), + itemCounts: getObjectCount, + crrStats: crrStatsObj, + repStatus: results.getReplicationStates, + config: cleanup(config), + capabilities: getCapabilities(), + ingestStats: results.getIngestionInfo.metrics, + ingestStatus: results.getIngestionInfo.status, + vaultReport: results.getVaultReport, + }; + monitoring.crrCacheToProm(results); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.write(JSON.stringify(response)); + log.end().debug('report handler finished'); + } + res.end(); + }, + ); } module.exports = { diff --git a/lib/utilities/request.js b/lib/utilities/request.js index 7be85927a6..7b5b09e4af 100644 --- a/lib/utilities/request.js +++ b/lib/utilities/request.js @@ -5,9 +5,7 @@ const { HttpProxyAgent } = require('http-proxy-agent'); const { HttpsProxyAgent } = require('https-proxy-agent'); const { jsutil } = require('arsenal'); -const { - proxyCompareUrl, -} = require('arsenal').storage.data.external.backendUtils; +const { proxyCompareUrl } = require('arsenal').storage.data.external.backendUtils; const validVerbs = new Set(['HEAD', 'GET', 'POST', 'PUT', 'DELETE']); const updateVerbs = new Set(['POST', 'PUT']); @@ -16,7 +14,7 @@ const updateVerbs = new Set(['POST', 'PUT']); * create a new header object from an existing header object. Similar keys * will be ignored if a value has been set for theirlower-cased form */ -function createHeaders(headers) { +function createHeaders(headers) { if (typeof headers !== 'object') { return {}; } @@ -72,7 +70,7 @@ function request(endpoint, options, callback) { let reqParams; if (typeof endpoint === 'string') { try { - reqParams = url.parse(endpoint); + reqParams = url.parse(endpoint); } catch (error) { return cb(error); } @@ -115,8 +113,10 @@ function request(endpoint, options, callback) { const req = request.request(reqParams); req.on('error', cb); req.on('response', res => { - const rawData = []; - res.on('data', chunk => { rawData.push(chunk); }); + const rawData = []; + res.on('data', chunk => { + rawData.push(chunk); + }); res.on('end', () => { const data = rawData.join(''); if (res.statusCode >= 400) { diff --git a/lib/utilities/serverAccessLogger.js b/lib/utilities/serverAccessLogger.js index 23c134eba7..98c3526f22 100644 --- a/lib/utilities/serverAccessLogger.js +++ b/lib/utilities/serverAccessLogger.js @@ -17,7 +17,9 @@ class ServerAccessLogger { this._terminated = false; this._reopenStream(); - setInterval(() => { this._checkFileRotated(); }, checkFileRotationIntervalMS); + setInterval(() => { + this._checkFileRotated(); + }, checkFileRotationIntervalMS); process.on('beforeExit', () => { this._cleanOldStream(true); @@ -48,7 +50,7 @@ class ServerAccessLogger { if (err) { if (err.code === 'ENOENT') { - logger.info('ServerAccessLogger: log file doesn\'t exist, creating a new one'); + logger.info("ServerAccessLogger: log file doesn't exist, creating a new one"); this._reopenStream(); return; } @@ -86,7 +88,9 @@ class ServerAccessLogger { fs.open(this._filename, 'a', 0o644, (err, fd) => { if (err) { logger.error('ServerAccessLogger: failed to reopenStream: open error', err); - setTimeout(() => { this._reopenStream(); }, this._retryReopenDelayMS); + setTimeout(() => { + this._reopenStream(); + }, this._retryReopenDelayMS); return; } @@ -100,7 +104,9 @@ class ServerAccessLogger { logger.error('ServerAccessLogger: failed to reopenStream: stat error', err); fs.close(fd); - setTimeout(() => { this._reopenStream(); }, this._retryReopenDelayMS); + setTimeout(() => { + this._reopenStream(); + }, this._retryReopenDelayMS); return; } @@ -144,10 +150,12 @@ class ServerAccessLogger { if (!this.stream.write(data)) { logger.warn('ServerAccessLogger: backpressure buffer full'); this._waitingDrain = true; - this.stream.once('drain', () => { this._waitingDrain = false; }); + this.stream.once('drain', () => { + this._waitingDrain = false; + }); } } -}; +} function setServerAccessLogger(logger) { serverAccessLogger = logger; @@ -183,7 +191,8 @@ function getRemoteIPFromRequest(request) { let remoteIP = null; if (request.headers) { // Check for forwarded IP headers (proxy/load balancer scenarios) - const headerRemoteIP = request.headers['x-forwarded-for'] || + const headerRemoteIP = + request.headers['x-forwarded-for'] || request.headers['x-real-ip'] || request.headers['x-client-ip'] || request.headers['cf-connecting-ip']; // Cloudflare @@ -196,9 +205,10 @@ function getRemoteIPFromRequest(request) { // Fallback to connection remote address if no forwarded headers if (!remoteIP) { - const connIP = (request.connection && request.connection.remoteAddress) || + const connIP = + (request.connection && request.connection.remoteAddress) || (request.socket && request.socket.remoteAddress) || - (request.ip); + request.ip; if (connIP) { remoteIP = connIP; } @@ -211,77 +221,77 @@ function getRemoteIPFromRequest(request) { // https://github.com/open-io/swift/blob/ff518e9907f74b5a2565973a260f36386b5d5cbf/etc/s3-default.cfg.in#L78 // https://stackoverflow.com/questions/42707878/amazon-s3-logs-operation-definition const methodToResType = Object.freeze({ - 'bucketDelete': 'BUCKET', - 'bucketDeleteCors': 'CORS', - 'bucketDeleteEncryption': 'ENCRYPTION', - 'bucketDeleteWebsite': 'WEBSITE', - 'bucketGet': 'BUCKET', - 'bucketGetACL': 'ACL', - 'bucketGetCors': 'CORS', - 'bucketGetObjectLock': 'OBJECT', - 'bucketGetVersioning': 'VERSIONING', - 'bucketGetWebsite': 'WEBSITE', - 'bucketGetLocation': 'LOCATION', - 'bucketGetEncryption': 'ENCRYPTION', - 'bucketHead': 'BUCKET', - 'bucketPut': 'BUCKET', - 'bucketPutACL': 'ACL', - 'bucketPutCors': 'CORS', - 'bucketPutVersioning': 'VERSIONING', - 'bucketPutTagging': 'TAGGING', - 'bucketDeleteTagging': 'TAGGING', - 'bucketGetTagging': 'TAGGING', - 'bucketPutWebsite': 'WEBSITE', - 'bucketPutReplication': 'REPLICATION', - 'bucketGetReplication': 'REPLICATION', - 'bucketDeleteReplication': 'REPLICATION', - 'bucketDeleteQuota': 'QUOTA', - 'bucketPutLifecycle': 'LIFECYCLE', - 'bucketUpdateQuota': 'QUOTA', - 'bucketGetLifecycle': 'LIFECYCLE', - 'bucketDeleteLifecycle': 'LIFECYCLE', - 'bucketPutPolicy': 'BUCKETPOLICY', - 'bucketGetPolicy': 'BUCKETPOLICY', - 'bucketGetQuota': 'QUOTA', - 'bucketDeletePolicy': 'BUCKETPOLICY', - 'bucketPutObjectLock': 'OBJECT', - 'bucketPutNotification': 'NOTIFICATION', - 'bucketGetNotification': 'NOTIFICATION', - 'bucketPutEncryption': 'ENCRYPTION', - 'bucketPutLogging': 'LOGGING_STATUS', - 'bucketGetLogging': 'LOGGING_STATUS', - 'bucketPutRateLimit': 'RATELIMIT', - 'bucketGetRateLimit': 'RATELIMIT', - 'bucketDeleteRateLimit': 'RATELIMIT', - 'corsPreflight': 'PREFLIGHT', - 'completeMultipartUpload': 'UPLOAD', - 'initiateMultipartUpload': 'UPLOADS', - 'listMultipartUploads': 'UPLOADS', - 'listParts': 'UPLOAD', - 'metadataSearch': 'OBJECT', - 'multiObjectDelete': 'MULTI_OBJECT_DELETE', - 'multipartDelete': 'UPLOAD', - 'objectDelete': 'OBJECT', - 'objectDeleteTagging': 'TAGGING', - 'objectGet': 'OBJECT', - 'objectGetAttributes': 'OBJECT', - 'objectGetACL': 'ACL', - 'objectGetLegalHold': 'LEGALHOLD', - 'objectGetRetention': 'OBJECT_LOCK_RETENTION', - 'objectGetTagging': 'TAGGING', - 'objectCopy': 'COPY', - 'objectHead': 'OBJECT', - 'objectPut': 'OBJECT', - 'objectPutACL': 'ACL', - 'objectPutLegalHold': 'LEGALHOLD', - 'objectPutTagging': 'TAGGING', - 'objectPutPart': 'PART', - 'objectPutCopyPart': 'COPY', - 'objectPutRetention': 'OBJECT_LOCK_RETENTION', - 'objectRestore': 'OBJECT', - 'serviceGet': 'SERVICE', // ListBuckets - 'websiteGet': 'WEBSITE', - 'websiteHead': 'WEBSITE', + bucketDelete: 'BUCKET', + bucketDeleteCors: 'CORS', + bucketDeleteEncryption: 'ENCRYPTION', + bucketDeleteWebsite: 'WEBSITE', + bucketGet: 'BUCKET', + bucketGetACL: 'ACL', + bucketGetCors: 'CORS', + bucketGetObjectLock: 'OBJECT', + bucketGetVersioning: 'VERSIONING', + bucketGetWebsite: 'WEBSITE', + bucketGetLocation: 'LOCATION', + bucketGetEncryption: 'ENCRYPTION', + bucketHead: 'BUCKET', + bucketPut: 'BUCKET', + bucketPutACL: 'ACL', + bucketPutCors: 'CORS', + bucketPutVersioning: 'VERSIONING', + bucketPutTagging: 'TAGGING', + bucketDeleteTagging: 'TAGGING', + bucketGetTagging: 'TAGGING', + bucketPutWebsite: 'WEBSITE', + bucketPutReplication: 'REPLICATION', + bucketGetReplication: 'REPLICATION', + bucketDeleteReplication: 'REPLICATION', + bucketDeleteQuota: 'QUOTA', + bucketPutLifecycle: 'LIFECYCLE', + bucketUpdateQuota: 'QUOTA', + bucketGetLifecycle: 'LIFECYCLE', + bucketDeleteLifecycle: 'LIFECYCLE', + bucketPutPolicy: 'BUCKETPOLICY', + bucketGetPolicy: 'BUCKETPOLICY', + bucketGetQuota: 'QUOTA', + bucketDeletePolicy: 'BUCKETPOLICY', + bucketPutObjectLock: 'OBJECT', + bucketPutNotification: 'NOTIFICATION', + bucketGetNotification: 'NOTIFICATION', + bucketPutEncryption: 'ENCRYPTION', + bucketPutLogging: 'LOGGING_STATUS', + bucketGetLogging: 'LOGGING_STATUS', + bucketPutRateLimit: 'RATELIMIT', + bucketGetRateLimit: 'RATELIMIT', + bucketDeleteRateLimit: 'RATELIMIT', + corsPreflight: 'PREFLIGHT', + completeMultipartUpload: 'UPLOAD', + initiateMultipartUpload: 'UPLOADS', + listMultipartUploads: 'UPLOADS', + listParts: 'UPLOAD', + metadataSearch: 'OBJECT', + multiObjectDelete: 'MULTI_OBJECT_DELETE', + multipartDelete: 'UPLOAD', + objectDelete: 'OBJECT', + objectDeleteTagging: 'TAGGING', + objectGet: 'OBJECT', + objectGetAttributes: 'OBJECT', + objectGetACL: 'ACL', + objectGetLegalHold: 'LEGALHOLD', + objectGetRetention: 'OBJECT_LOCK_RETENTION', + objectGetTagging: 'TAGGING', + objectCopy: 'COPY', + objectHead: 'OBJECT', + objectPut: 'OBJECT', + objectPutACL: 'ACL', + objectPutLegalHold: 'LEGALHOLD', + objectPutTagging: 'TAGGING', + objectPutPart: 'PART', + objectPutCopyPart: 'COPY', + objectPutRetention: 'OBJECT_LOCK_RETENTION', + objectRestore: 'OBJECT', + serviceGet: 'SERVICE', // ListBuckets + websiteGet: 'WEBSITE', + websiteHead: 'WEBSITE', }); function getOperation(req) { @@ -329,7 +339,7 @@ function getOperation(req) { process.emitWarning('Unknown apiMethod for server access log', { type: 'ServerAccessLogWarning', code: 'UNKNOWN_API_METHOD', - detail: `apiMethod=${req.apiMethod}, method=${req.method}, url=${req.url}` + detail: `apiMethod=${req.apiMethod}, method=${req.method}, url=${req.url}`, }); } return `REST.${req.method}.UNKNOWN`; @@ -373,12 +383,12 @@ function getURI(request) { } const objectSizePutMethods = Object.freeze({ - 'objectPut': true, - 'objectPutPart': true, + objectPut: true, + objectPutPart: true, }); const objectSizeGetMethods = Object.freeze({ - 'objectGet': true, + objectGet: true, }); function getObjectSize(request, response) { @@ -440,8 +450,12 @@ function calculateTotalTime(startTime, onFinishEndTime) { } function calculateTurnAroundTime(startTurnAroundTime, endTurnAroundTime) { - if (startTurnAroundTime === undefined || startTurnAroundTime === null - || endTurnAroundTime === undefined || endTurnAroundTime === null) { + if ( + startTurnAroundTime === undefined || + startTurnAroundTime === null || + endTurnAroundTime === undefined || + endTurnAroundTime === null + ) { return null; } @@ -509,12 +523,12 @@ function buildLogEntry(req, params, options) { signatureVersion: authInfo?.getAuthVersion() ?? undefined, cipherSuite: req.socket?.encrypted ? req.socket.getCipher()['standardName'] - : req.headers?.['x-ssl-cipher'] ?? undefined, + : (req.headers?.['x-ssl-cipher'] ?? undefined), authenticationType: authInfo?.getAuthType() ?? undefined, hostHeader: req.headers?.host ?? undefined, tlsVersion: req.socket?.encrypted ? req.socket.getCipher()['version'] - : req.headers?.['x-ssl-protocol'] ?? undefined, + : (req.headers?.['x-ssl-protocol'] ?? undefined), aclRequired: options.aclRequired ?? undefined, // hostID: undefined, // NOT IMPLEMENTED // accessPointARN: undefined, // NOT IMPLEMENTED @@ -525,7 +539,7 @@ function buildLogEntry(req, params, options) { // eslint-disable-next-line camelcase req_id: options.requestID ?? undefined, // AWS "Request ID" field bytesSent: options.bytesSent ?? undefined, - clientIP: getRemoteIPFromRequest(req) ?? undefined, // AWS 'Remote IP' field + clientIP: getRemoteIPFromRequest(req) ?? undefined, // AWS 'Remote IP' field httpCode: options.httpCode ?? undefined, // AWS "HTTP Status" field objectKey: options.objectKey ?? undefined, // AWS "Key" field @@ -533,8 +547,8 @@ function buildLogEntry(req, params, options) { logFormatVersion: SERVER_ACCESS_LOG_FORMAT_VERSION, // For backbeat requests other than expiration and replication, // force loggingEnabled to false to prevent delivery to log courier. - loggingEnabled: (params.backbeat && !params.expiration && !params.replication) - ? false : (params.enabled ?? undefined), + loggingEnabled: + params.backbeat && !params.expiration && !params.replication ? false : (params.enabled ?? undefined), loggingTargetBucket: params.loggingEnabled?.TargetBucket ?? undefined, loggingTargetPrefix: params.loggingEnabled?.TargetPrefix ?? undefined, awsAccessKeyID: authInfo?.getAccessKey() ?? undefined, @@ -598,9 +612,8 @@ function logServerAccess(req, res) { const logEntry = buildLogEntry(req, params, { bytesDeleted: params.analyticsBytesDeleted, bytesReceived: Number.isInteger(req.parsedContentLength) ? req.parsedContentLength : undefined, - bodyLength: req.headers['content-length'] !== undefined - ? parseInt(req.headers['content-length'], 10) - : undefined, + bodyLength: + req.headers['content-length'] !== undefined ? parseInt(req.headers['content-length'], 10) : undefined, contentLength: getObjectSize(req, res), // eslint-disable-next-line camelcase elapsed_ms: calculateElapsedMS(params.startTime, params.onCloseEndTime), @@ -669,8 +682,7 @@ function logServerAccess(req, res) { query = versionId ? `?acl&versionId=${versionId}` : '?acl'; } const encodedKey = params.objectKey.split('/').map(encodeURIComponent).join('/'); - logEntry.requestURI = - `${method} /${params.bucketName}/${encodedKey}${query} HTTP/${req.httpVersion ?? '1.1'}`; + logEntry.requestURI = `${method} /${params.bucketName}/${encodedKey}${query} HTTP/${req.httpVersion ?? '1.1'}`; } if (params.internalLogRequestQueue && params.internalLogRequestQueue.length > 0) { @@ -680,8 +692,14 @@ function logServerAccess(req, res) { } } else if (logEntry.operation.includes('COPY')) { for (const entry of params.internalLogRequestQueue) { - logCopySourceAccess(req, requestID, entry.operation, entry.sourceBucket, - entry.sourceObject, entry.objectSize); + logCopySourceAccess( + req, + requestID, + entry.operation, + entry.sourceBucket, + entry.sourceObject, + entry.objectSize, + ); } } } diff --git a/lib/utilities/validateQueryAndHeaders.js b/lib/utilities/validateQueryAndHeaders.js index 376e3c7c24..37c07727ba 100644 --- a/lib/utilities/validateQueryAndHeaders.js +++ b/lib/utilities/validateQueryAndHeaders.js @@ -14,7 +14,6 @@ function _validateKeys(unsupportedKeys, obj) { return unsupportedKey; } - /** * validateQueryAndHeaders - Check request for unsupported queries or headers * @param {object} request - request object @@ -28,8 +27,7 @@ function validateQueryAndHeaders(request, log) { const isBucketQuery = !request.objectKey; // if the request is at bucket level, check for unsupported bucket queries if (isBucketQuery) { - const unsupportedQuery = - _validateKeys(constants.unsupportedBucketQueries, reqQuery); + const unsupportedQuery = _validateKeys(constants.unsupportedBucketQueries, reqQuery); if (unsupportedQuery) { log.debug('encountered unsupported query', { query: unsupportedQuery, @@ -38,8 +36,7 @@ function validateQueryAndHeaders(request, log) { return { error: errors.NotImplemented }; } } - const unsupportedQuery = _validateKeys(constants.unsupportedQueries, - reqQuery); + const unsupportedQuery = _validateKeys(constants.unsupportedQueries, reqQuery); if (unsupportedQuery) { log.debug('encountered unsupported query', { query: unsupportedQuery, @@ -47,8 +44,7 @@ function validateQueryAndHeaders(request, log) { }); return { error: errors.NotImplemented }; } - const unsupportedHeader = _validateKeys(constants.unsupportedHeaders, - reqHeaders); + const unsupportedHeader = _validateKeys(constants.unsupportedHeaders, reqHeaders); if (unsupportedHeader) { log.debug('encountered unsupported header', { header: unsupportedHeader, diff --git a/lib/utilization/scuba/wrapper.js b/lib/utilization/scuba/wrapper.js index 0cde306868..43201824db 100644 --- a/lib/utilization/scuba/wrapper.js +++ b/lib/utilization/scuba/wrapper.js @@ -27,28 +27,30 @@ class ScubaClientImpl extends ScubaClient { } _healthCheck() { - return this.healthCheck().then(data => { - if (data?.date) { - const date = new Date(data.date); - if (Date.now() - date.getTime() > this.maxStaleness) { - throw new Error('Data is stale, disabling quotas'); + return this.healthCheck() + .then(data => { + if (data?.date) { + const date = new Date(data.date); + if (Date.now() - date.getTime() > this.maxStaleness) { + throw new Error('Data is stale, disabling quotas'); + } } - } - if (!this.enabled) { - this._log.info('Scuba health check passed, enabling quotas'); - } - monitoring.utilizationServiceAvailable.set(1); - this.enabled = true; - }).catch(err => { - if (this.enabled) { - this._log.warn('Scuba health check failed, disabling quotas', { - err: err.name, - description: err.message, - }); - } - monitoring.utilizationServiceAvailable.set(0); - this.enabled = false; - }); + if (!this.enabled) { + this._log.info('Scuba health check passed, enabling quotas'); + } + monitoring.utilizationServiceAvailable.set(1); + this.enabled = true; + }) + .catch(err => { + if (this.enabled) { + this._log.warn('Scuba health check failed, disabling quotas', { + err: err.name, + description: err.message, + }); + } + monitoring.utilizationServiceAvailable.set(0); + this.enabled = false; + }); } periodicHealthCheck() { @@ -56,20 +58,24 @@ class ScubaClientImpl extends ScubaClient { clearInterval(this._healthCheckTimer); } this._healthCheck(); - this._healthCheckTimer = setInterval(async () => { - this._healthCheck(); - }, Number(process.env.SCUBA_HEALTHCHECK_FREQUENCY) - || externalBackendHealthCheckInterval); + this._healthCheckTimer = setInterval( + async () => { + this._healthCheck(); + }, + Number(process.env.SCUBA_HEALTHCHECK_FREQUENCY) || externalBackendHealthCheckInterval, + ); } getUtilizationMetrics(metricsClass, resourceName, options, body, callback) { const requestStartTime = process.hrtime.bigint(); return this._getLatestMetricsCallback(metricsClass, resourceName, options, body, (err, data) => { const responseTimeInNs = Number(process.hrtime.bigint() - requestStartTime); - monitoring.utilizationMetricsRetrievalDuration.labels({ - code: err ? (err.statusCode || 500) : 200, - class: metricsClass, - }).observe(responseTimeInNs / 1e9); + monitoring.utilizationMetricsRetrievalDuration + .labels({ + code: err ? err.statusCode || 500 : 200, + class: metricsClass, + }) + .observe(responseTimeInNs / 1e9); return callback(err, data); }); } diff --git a/managementAgent.js b/managementAgent.js index d7161ef693..4413d36c3b 100644 --- a/managementAgent.js +++ b/managementAgent.js @@ -8,13 +8,11 @@ const { managementAgentMessageType } = require('./lib/management/agentClient'); const { addOverlayMessageListener } = require('./lib/management/push'); const { saveConfigurationVersion } = require('./lib/management/configuration'); - // TODO: auth? // TODO: werelogs with a specific name. const CHECK_BROKEN_CONNECTIONS_FREQUENCY_MS = 15000; - class ManagementAgentServer { constructor() { this.port = _config.managementAgent.port || 8010; @@ -34,9 +32,7 @@ class ManagementAgentServer { /* Define REPORT_TOKEN env variable needed by the management * module. */ - process.env.REPORT_TOKEN = process.env.REPORT_TOKEN - || _config.reportToken - || Uuid.v4(); + process.env.REPORT_TOKEN = process.env.REPORT_TOKEN || _config.reportToken || Uuid.v4(); initManagement(logger.newRequestLogger(), overlay => { let error = null; @@ -73,8 +69,7 @@ class ManagementAgentServer { this.wss.on('listening', this.onListening.bind(this)); this.wss.on('error', this.onError.bind(this)); - setInterval(this.checkBrokenConnections.bind(this), - CHECK_BROKEN_CONNECTIONS_FREQUENCY_MS); + setInterval(this.checkBrokenConnections.bind(this), CHECK_BROKEN_CONNECTIONS_FREQUENCY_MS); addOverlayMessageListener(this.onNewOverlay.bind(this)); } @@ -114,8 +109,7 @@ class ManagementAgentServer { } onListening() { - logger.info('websocket server listening', - { port: this.port }); + logger.info('websocket server listening', { port: this.port }); } onError(error) { @@ -137,27 +131,24 @@ class ManagementAgentServer { }; client.send(JSON.stringify(msg), error => { if (error) { - logger.error( - 'failed to send remoteOverlay to management agent client', { - error, client: client._socket._peername, - }); + logger.error('failed to send remoteOverlay to management agent client', { + error, + client: client._socket._peername, + }); } }); } onNewOverlay(remoteOverlay) { const remoteOverlayObj = JSON.parse(remoteOverlay); - saveConfigurationVersion( - this.loadedOverlay, remoteOverlayObj, logger, err => { - if (err) { - logger.error('failed to save remote overlay', { err }); - return; - } - this.loadedOverlay = remoteOverlayObj; - this.wss.clients.forEach( - this._sendNewOverlayToClient.bind(this) - ); - }); + saveConfigurationVersion(this.loadedOverlay, remoteOverlayObj, logger, err => { + if (err) { + logger.error('failed to save remote overlay', { err }); + return; + } + this.loadedOverlay = remoteOverlayObj; + this.wss.clients.forEach(this._sendNewOverlayToClient.bind(this)); + }); } checkBrokenConnections() { diff --git a/mdserver.js b/mdserver.js index 4244be1cbc..c5113a46a3 100644 --- a/mdserver.js +++ b/mdserver.js @@ -1,8 +1,7 @@ 'use strict'; const { config } = require('./lib/Config.js'); -const MetadataFileServer = - require('arsenal').storage.metadata.file.MetadataFileServer; +const MetadataFileServer = require('arsenal').storage.metadata.file.MetadataFileServer; const logger = require('./lib/utilities/logger'); process.on('uncaughtException', err => { @@ -16,14 +15,15 @@ process.on('uncaughtException', err => { }); if (config.backends.metadata === 'file') { - const mdServer = new MetadataFileServer( - { bindAddress: config.metadataDaemon.bindAddress, - port: config.metadataDaemon.port, - path: config.metadataDaemon.metadataPath, - restEnabled: config.metadataDaemon.restEnabled, - restPort: config.metadataDaemon.restPort, - recordLog: config.recordLog, - versioning: { replicationGroupId: config.replicationGroupId }, - log: config.log }); + const mdServer = new MetadataFileServer({ + bindAddress: config.metadataDaemon.bindAddress, + port: config.metadataDaemon.port, + path: config.metadataDaemon.metadataPath, + restEnabled: config.metadataDaemon.restEnabled, + restPort: config.metadataDaemon.restPort, + recordLog: config.recordLog, + versioning: { replicationGroupId: config.replicationGroupId }, + log: config.log, + }); mdServer.startServer(); } diff --git a/monitoring/alerts.yaml b/monitoring/alerts.yaml index fc1621ba79..6c1f1b0a6e 100644 --- a/monitoring/alerts.yaml +++ b/monitoring/alerts.yaml @@ -34,149 +34,148 @@ x-inputs: value: 0.500 groups: -- name: CloudServer - rules: + - name: CloudServer + rules: + - alert: DataAccessS3EndpointDegraded + expr: sum(up{namespace="${namespace}", service="${service}"}) < ${replicas} + for: '30s' + labels: + severity: warning + annotations: + description: 'Less than 100% of S3 endpoints are up and healthy' + summary: 'Data Access service is degraded' - - alert: DataAccessS3EndpointDegraded - expr: sum(up{namespace="${namespace}", service="${service}"}) < ${replicas} - for: "30s" - labels: - severity: warning - annotations: - description: "Less than 100% of S3 endpoints are up and healthy" - summary: "Data Access service is degraded" + - alert: DataAccessS3EndpointCritical + expr: sum(up{namespace="${namespace}", service="${service}"}) * 2 < ${replicas} + for: '30s' + labels: + severity: critical + annotations: + description: 'Less than 50% of S3 endpoints are up and healthy' + summary: 'Data Access service is critical' - - alert: DataAccessS3EndpointCritical - expr: sum(up{namespace="${namespace}", service="${service}"}) * 2 < ${replicas} - for: "30s" - labels: - severity: critical - annotations: - description: "Less than 50% of S3 endpoints are up and healthy" - summary: "Data Access service is critical" + # As a platform admin I want to be alerted (warning) when the system errors are more than 3% of + # all the response codes + - alert: SystemErrorsWarning + expr: | + sum(rate(s3_cloudserver_http_requests_total{namespace="${namespace}", service="${service}", code=~"5.."}[1m])) + / sum(rate(s3_cloudserver_http_requests_total{namespace="${namespace}", service="${service}"}[1m])) + >= ${systemErrorsWarningThreshold} + for: 5m + labels: + severity: warning + annotations: + description: 'System errors represent more than 3% of all the response codes' + summary: 'High ratio of system erors' - # As a platform admin I want to be alerted (warning) when the system errors are more than 3% of - # all the response codes - - alert: SystemErrorsWarning - expr: | - sum(rate(s3_cloudserver_http_requests_total{namespace="${namespace}", service="${service}", code=~"5.."}[1m])) - / sum(rate(s3_cloudserver_http_requests_total{namespace="${namespace}", service="${service}"}[1m])) - >= ${systemErrorsWarningThreshold} - for: 5m - labels: - severity: warning - annotations: - description: "System errors represent more than 3% of all the response codes" - summary: "High ratio of system erors" + # As a platform admin I want to be alerted (critical) when the system errors are more than 5% of + # all the response codes + - alert: SystemErrorsCritical + expr: | + sum(rate(s3_cloudserver_http_requests_total{namespace="${namespace}", service="${service}", code=~"5.."}[1m])) + / sum(rate(s3_cloudserver_http_requests_total{namespace="${namespace}", service="${service}"}[1m])) + >= ${systemErrorsCriticalThreshold} + for: 5m + labels: + severity: critical + annotations: + description: 'System errors represent more than 5% of all the response codes' + summary: 'Very high ratio of system erors' - # As a platform admin I want to be alerted (critical) when the system errors are more than 5% of - # all the response codes - - alert: SystemErrorsCritical - expr: | - sum(rate(s3_cloudserver_http_requests_total{namespace="${namespace}", service="${service}", code=~"5.."}[1m])) - / sum(rate(s3_cloudserver_http_requests_total{namespace="${namespace}", service="${service}"}[1m])) - >= ${systemErrorsCriticalThreshold} - for: 5m - labels: - severity: critical - annotations: - description: "System errors represent more than 5% of all the response codes" - summary: "Very high ratio of system erors" + # As a platform admin I want to be alerted (warning) when a listing operation latency or a + # version listing operation latency is more than 300ms + - alert: ListingLatencyWarning + expr: | + sum(rate(s3_cloudserver_http_request_duration_seconds_sum{namespace="${namespace}",service="${service}",action="listBucket"}[1m])) + / sum(rate(s3_cloudserver_http_request_duration_seconds_count{namespace="${namespace}",service="${service}",action="listBucket"}[1m])) + >= ${listingLatencyWarningThreshold} + for: 5m + labels: + severity: warning + annotations: + description: 'Latency of listing or version listing operations is more than 300ms' + summary: 'High listing latency' - # As a platform admin I want to be alerted (warning) when a listing operation latency or a - # version listing operation latency is more than 300ms - - alert: ListingLatencyWarning - expr: | - sum(rate(s3_cloudserver_http_request_duration_seconds_sum{namespace="${namespace}",service="${service}",action="listBucket"}[1m])) - / sum(rate(s3_cloudserver_http_request_duration_seconds_count{namespace="${namespace}",service="${service}",action="listBucket"}[1m])) - >= ${listingLatencyWarningThreshold} - for: 5m - labels: - severity: warning - annotations: - description: "Latency of listing or version listing operations is more than 300ms" - summary: "High listing latency" + # As a platform admin I want to be alerted (critical) when a listing operation latency or a + # version listing operation latency is more than 500ms + - alert: ListingLatencyCritical + expr: | + sum(rate(s3_cloudserver_http_request_duration_seconds_sum{namespace="${namespace}",service="${service}",action="listBucket"}[1m])) + / sum(rate(s3_cloudserver_http_request_duration_seconds_count{namespace="${namespace}",service="${service}",action="listBucket"}[1m])) + >= ${listingLatencyCriticalThreshold} + for: 5m + labels: + severity: critical + annotations: + description: 'Latency of listing or version listing operations is more than 500ms' + summary: 'Very high listing latency' - # As a platform admin I want to be alerted (critical) when a listing operation latency or a - # version listing operation latency is more than 500ms - - alert: ListingLatencyCritical - expr: | - sum(rate(s3_cloudserver_http_request_duration_seconds_sum{namespace="${namespace}",service="${service}",action="listBucket"}[1m])) - / sum(rate(s3_cloudserver_http_request_duration_seconds_count{namespace="${namespace}",service="${service}",action="listBucket"}[1m])) - >= ${listingLatencyCriticalThreshold} - for: 5m - labels: - severity: critical - annotations: - description: "Latency of listing or version listing operations is more than 500ms" - summary: "Very high listing latency" + # As a platform admin I want to be alerted (warning) when a delete operation latency is more than + # 500ms + - alert: DeleteLatencyWarning + expr: | + sum(rate(s3_cloudserver_http_request_duration_seconds_sum{namespace="${namespace}",service="${service}",action="deleteObject"}[1m])) + / sum(rate(s3_cloudserver_http_request_duration_seconds_count{namespace="${namespace}",service="${service}",action="deleteObject"}[1m])) + >= ${deleteLatencyWarningThreshold} + for: 5m + labels: + severity: warning + annotations: + description: 'Latency of delete object operations is more than 500ms' + summary: 'High delete latency' - # As a platform admin I want to be alerted (warning) when a delete operation latency is more than - # 500ms - - alert: DeleteLatencyWarning - expr: | - sum(rate(s3_cloudserver_http_request_duration_seconds_sum{namespace="${namespace}",service="${service}",action="deleteObject"}[1m])) - / sum(rate(s3_cloudserver_http_request_duration_seconds_count{namespace="${namespace}",service="${service}",action="deleteObject"}[1m])) - >= ${deleteLatencyWarningThreshold} - for: 5m - labels: - severity: warning - annotations: - description: "Latency of delete object operations is more than 500ms" - summary: "High delete latency" + # As a platform admin I want to be alerted (critical) when a delete operation latency is more + # than 1s + - alert: DeleteLatencyCritical + expr: | + sum(rate(s3_cloudserver_http_request_duration_seconds_sum{namespace="${namespace}",service="${service}",action="deleteObject"}[1m])) + / sum(rate(s3_cloudserver_http_request_duration_seconds_count{namespace="${namespace}",service="${service}",action="deleteObject"}[1m])) + >= ${deleteLatencyCriticalThreshold} + for: 5m + labels: + severity: critical + annotations: + description: 'Latency of delete object operations is more than 1s' + summary: 'Very high delete latency' - # As a platform admin I want to be alerted (critical) when a delete operation latency is more - # than 1s - - alert: DeleteLatencyCritical - expr: | - sum(rate(s3_cloudserver_http_request_duration_seconds_sum{namespace="${namespace}",service="${service}",action="deleteObject"}[1m])) - / sum(rate(s3_cloudserver_http_request_duration_seconds_count{namespace="${namespace}",service="${service}",action="deleteObject"}[1m])) - >= ${deleteLatencyCriticalThreshold} - for: 5m - labels: - severity: critical - annotations: - description: "Latency of delete object operations is more than 1s" - summary: "Very high delete latency" + # As a platform admin I want to be alerted (warning) when the utilization metrics service is enabled + # but not available for at least half of the S3 services during the last minute + - alert: QuotaMetricsNotAvailable + expr: | + avg(s3_cloudserver_quota_utilization_service_available{namespace="${namespace}",service="${service}"}) + < ${quotaUnavailabilityThreshold} and + (max(s3_cloudserver_quota_buckets_count{namespace="${namespace}", job="${reportJob}"}) > 0 or + max(s3_cloudserver_quota_accounts_count{namespace="${namespace}", job="${reportJob}"}) > 0) + labels: + severity: warning + annotations: + description: 'The storage metrics required for Account or S3 Bucket Quota checks are not available, the quotas are disabled.' + summary: 'Utilization metrics service not available' - # As a platform admin I want to be alerted (warning) when the utilization metrics service is enabled - # but not available for at least half of the S3 services during the last minute - - alert: QuotaMetricsNotAvailable - expr: | - avg(s3_cloudserver_quota_utilization_service_available{namespace="${namespace}",service="${service}"}) - < ${quotaUnavailabilityThreshold} and - (max(s3_cloudserver_quota_buckets_count{namespace="${namespace}", job="${reportJob}"}) > 0 or - max(s3_cloudserver_quota_accounts_count{namespace="${namespace}", job="${reportJob}"}) > 0) - labels: - severity: warning - annotations: - description: "The storage metrics required for Account or S3 Bucket Quota checks are not available, the quotas are disabled." - summary: "Utilization metrics service not available" + # As a platform admin I want to be alerted (critical) when the utilization metrics service is enabled + # but not available during the last 10 minutes + - alert: QuotaMetricsNotAvailable + expr: | + avg(s3_cloudserver_quota_utilization_service_available{namespace="${namespace}",service="${service}"}) + < ${quotaUnavailabilityThreshold} and + (max(s3_cloudserver_quota_buckets_count{namespace="${namespace}", job="${reportJob}"}) > 0 or + max(s3_cloudserver_quota_accounts_count{namespace="${namespace}", job="${reportJob}"}) > 0) + for: 10m + labels: + severity: critical + annotations: + description: 'The storage metrics required for Account or S3 Bucket Quota checks are not available, the quotas are disabled.' + summary: 'Utilization metrics service not available' - # As a platform admin I want to be alerted (critical) when the utilization metrics service is enabled - # but not available during the last 10 minutes - - alert: QuotaMetricsNotAvailable - expr: | - avg(s3_cloudserver_quota_utilization_service_available{namespace="${namespace}",service="${service}"}) - < ${quotaUnavailabilityThreshold} and - (max(s3_cloudserver_quota_buckets_count{namespace="${namespace}", job="${reportJob}"}) > 0 or - max(s3_cloudserver_quota_accounts_count{namespace="${namespace}", job="${reportJob}"}) > 0) - for: 10m - labels: - severity: critical - annotations: - description: "The storage metrics required for Account or S3 Bucket Quota checks are not available, the quotas are disabled." - summary: "Utilization metrics service not available" - - # As a platform admin I want to be alerted (critical) when quotas were not honored due to metrics - # being unavailable - - alert: QuotaUnavailable - expr: | - sum(increase(s3_cloudserver_quota_unavailable_count{namespace="${namespace}",service="${service}"}[2m])) - > 0 - for: 5m - labels: - severity: critical - annotations: - description: "Quotas were not honored due to metrics being unavailable. If the S3 Bucket or Account was created recently, the metrics may not be available yet." - summary: "High number of quota requests with metrics unavailable" + # As a platform admin I want to be alerted (critical) when quotas were not honored due to metrics + # being unavailable + - alert: QuotaUnavailable + expr: | + sum(increase(s3_cloudserver_quota_unavailable_count{namespace="${namespace}",service="${service}"}[2m])) + > 0 + for: 5m + labels: + severity: critical + annotations: + description: 'Quotas were not honored due to metrics being unavailable. If the S3 Bucket or Account was created recently, the metrics may not be available yet.' + summary: 'High number of quota requests with metrics unavailable' diff --git a/monitoring/dashboard.json b/monitoring/dashboard.json index 1eca3701c1..7b6664a4b0 100644 --- a/monitoring/dashboard.json +++ b/monitoring/dashboard.json @@ -1,3629 +1,3528 @@ { - "__inputs": [ - { - "description": "", - "label": "Prometheus", - "name": "DS_PROMETHEUS", - "pluginId": "prometheus", - "pluginName": "Prometheus", - "type": "datasource" - }, - { - "description": "", - "label": "Loki", - "name": "DS_LOKI", - "pluginId": "loki", - "pluginName": "Loki", - "type": "datasource" - }, - { - "description": "Namespace associated with the Zenko instance", - "label": "namespace", - "name": "namespace", - "type": "constant", - "value": "zenko" - }, - { - "description": "Name of the Zenko instance", - "label": "instance", - "name": "zenkoName", - "type": "constant", - "value": "artesca-data" - }, - { - "description": "Name of the Cloudserver container, used to filter only the Cloudserver services.", - "label": "container", - "name": "container", - "type": "constant", - "value": "connector-cloudserver" - }, - { - "description": "Name of the Cloudserver Report job, used to filter only the Report Handler instances.", - "label": "report job", - "name": "reportJob", - "type": "constant", - "value": "artesca-data-ops-report-handler" - }, - { - "description": "Name of the Count-Items cronjob, used to filter only the Count-Items instances.", - "label": "count-items job", - "name": "countItemsJob", - "type": "constant", - "value": "artesca-data-ops-count-items" - } - ], - "annotations": { - "list": [] - }, - "description": "", - "editable": true, - "gnetId": null, - "hideControls": false, - "id": null, - "links": [], - "panels": [ - { - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": null, - "mappings": [], - "noValue": "none", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "index": 0, - "line": true, - "op": "gt", - "value": "null", - "yaxis": "left" - }, - { - "color": "green", - "index": 1, - "line": true, - "op": "gt", - "value": 1.0, - "yaxis": "left" - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 0, - "y": 0 - }, - "hideTimeOverride": false, - "id": 1, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ + "__inputs": [ { - "datasource": null, - "expr": "sum(up{namespace=\"${namespace}\", job=~\"$job\"})", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Up", - "transformations": [], - "transparent": false, - "type": "stat" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": null, - "mappings": [], - "noValue": "0", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "index": 0, - "line": true, - "op": "gt", - "value": "null", - "yaxis": "left" - } - ] - }, - "unit": "ops" + "description": "", + "label": "Prometheus", + "name": "DS_PROMETHEUS", + "pluginId": "prometheus", + "pluginName": "Prometheus", + "type": "datasource" }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 3, - "y": 0 - }, - "hideTimeOverride": false, - "id": 2, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ { - "datasource": null, - "expr": "sum(rate(s3_cloudserver_http_requests_total{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval]))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Http requests rate", - "transformations": [], - "transparent": false, - "type": "stat" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "calcs": [ - "mean" - ], - "decimals": null, - "limit": null, - "links": [], - "mappings": [], - "max": 100, - "min": 0, - "noValue": "-", - "override": {}, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#808080", - "index": 0, - "line": true, - "op": "gt", - "value": "null", - "yaxis": "left" - }, - { - "color": "red", - "index": 1, - "line": true, - "op": "gt", - "value": 0.0, - "yaxis": "left" - }, - { - "color": "orange", - "index": 2, - "line": true, - "op": "gt", - "value": 80.0, - "yaxis": "left" - }, - { - "color": "green", - "index": 3, - "line": true, - "op": "gt", - "value": 90.0, - "yaxis": "left" - } - ] - }, - "title": null, - "unit": "percent", - "values": false + "description": "", + "label": "Loki", + "name": "DS_LOKI", + "pluginId": "loki", + "pluginName": "Loki", + "type": "datasource" }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 6, - "y": 0 - }, - "hideTimeOverride": false, - "id": 3, - "links": [], - "maxDataPoints": 100, - "options": { - "reduceOptions": { - "calcs": [ - "mean" - ] - } - }, - "targets": [ { - "datasource": null, - "expr": "sum(rate(s3_cloudserver_http_requests_total{namespace=\"${namespace}\", job=~\"$job\", code=~\"2..\"}[$__rate_interval])) * 100\n /\nsum(rate(s3_cloudserver_http_requests_total{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval]) > 0)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Success rate", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Success rate", - "transformations": [], - "transparent": false, - "type": "gauge" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Rate of data ingested : cumulative amount of data created (>0) or deleted (<0) per second.", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": 1, - "mappings": [], - "noValue": "none", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "dark-purple", - "index": 0, - "line": true, - "op": "gt", - "value": "null", - "yaxis": "left" - } - ] - }, - "unit": "binBps" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 9, - "y": 0 - }, - "hideTimeOverride": false, - "id": 4, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "background", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false + "description": "Namespace associated with the Zenko instance", + "label": "namespace", + "name": "namespace", + "type": "constant", + "value": "zenko" }, - "textMode": "auto" - }, - "targets": [ { - "datasource": null, - "expr": "-sum(deriv(s3_cloudserver_disk_available_bytes{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval]))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Injection Data Rate", - "transformations": [], - "transparent": false, - "type": "stat" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Rate of object ingestion : cumulative count of object created (>0) or deleted (<0) per second.", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": 1, - "mappings": [], - "noValue": "none", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "dark-purple", - "index": 0, - "line": true, - "op": "gt", - "value": "null", - "yaxis": "left" - } - ] - }, - "unit": "O/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 12, - "y": 0 - }, - "hideTimeOverride": false, - "id": 5, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "background", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false + "description": "Name of the Zenko instance", + "label": "instance", + "name": "zenkoName", + "type": "constant", + "value": "artesca-data" }, - "textMode": "auto" - }, - "targets": [ { - "datasource": null, - "expr": "sum(deriv(s3_cloudserver_objects_count{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval]))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Injection Rate", - "transformations": [], - "transparent": false, - "type": "stat" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Number of S3 buckets available in the cluster.\nThis value is computed asynchronously, and update may be delayed up to 1h.", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": null, - "mappings": [], - "noValue": "-", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#808080", - "index": 0, - "line": true, - "op": "gt", - "value": "null", - "yaxis": "left" - }, - { - "color": "blue", - "index": 1, - "line": true, - "op": "gt", - "value": 0.0, - "yaxis": "left" - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 7, - "x": 15, - "y": 0 - }, - "hideTimeOverride": false, - "id": 6, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + "description": "Name of the Cloudserver container, used to filter only the Cloudserver services.", + "label": "container", + "name": "container", + "type": "constant", + "value": "connector-cloudserver" }, - "textMode": "auto" - }, - "targets": [ { - "datasource": null, - "expr": "sum(s3_cloudserver_buckets_count{namespace=\"${namespace}\", job=\"${reportJob}\"})", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Buckets", - "transformations": [], - "transparent": false, - "type": "stat" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Status of the reports-handler pod.", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": null, - "mappings": [], - "noValue": "none", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "index": 0, - "line": true, - "op": "gt", - "value": "null", - "yaxis": "left" - }, - { - "color": "green", - "index": 1, - "line": true, - "op": "gt", - "value": 1.0, - "yaxis": "left" - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 22, - "y": 0 - }, - "hideTimeOverride": false, - "id": 7, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false + "description": "Name of the Cloudserver Report job, used to filter only the Report Handler instances.", + "label": "report job", + "name": "reportJob", + "type": "constant", + "value": "artesca-data-ops-report-handler" }, - "textMode": "auto" - }, - "targets": [ { - "datasource": null, - "expr": "sum(up{namespace=\"${namespace}\", job=\"${reportJob}\"})", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "", - "step": 10, - "target": "" + "description": "Name of the Count-Items cronjob, used to filter only the Count-Items instances.", + "label": "count-items job", + "name": "countItemsJob", + "type": "constant", + "value": "artesca-data-ops-count-items" } - ], - "title": "Reporter", - "transformations": [], - "transparent": false, - "type": "stat" + ], + "annotations": { + "list": [] }, - { - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": null, - "mappings": [], - "noValue": "0", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "semi-dark-blue", - "index": 0, - "line": true, - "op": "gt", - "value": "null", - "yaxis": "left" - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 0, - "y": 4 - }, - "hideTimeOverride": false, - "id": 8, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "background", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ + "description": "", + "editable": true, + "gnetId": null, + "hideControls": false, + "id": null, + "links": [], + "panels": [ { - "datasource": null, - "expr": "sum(rate(s3_cloudserver_http_requests_total{namespace=\"${namespace}\", job=~\"$job\",code=\"200\"}[$__rate_interval]))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Status 200", - "transformations": [], - "transparent": false, - "type": "stat" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": null, - "mappings": [], - "noValue": "0", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "semi-dark-blue", - "index": 0, - "line": true, - "op": "gt", - "value": "null", - "yaxis": "left" - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 3, - "y": 4 - }, - "hideTimeOverride": false, - "id": 9, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "background", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": null, + "mappings": [], + "noValue": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "index": 0, + "line": true, + "op": "gt", + "value": "null", + "yaxis": "left" + }, + { + "color": "green", + "index": 1, + "line": true, + "op": "gt", + "value": 1.0, + "yaxis": "left" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 0 + }, + "hideTimeOverride": false, + "id": 1, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["last"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": null, + "expr": "sum(up{namespace=\"${namespace}\", job=~\"$job\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Up", + "transformations": [], + "transparent": false, + "type": "stat" }, - "textMode": "auto" - }, - "targets": [ { - "datasource": null, - "expr": "sum(rate(s3_cloudserver_http_requests_total{namespace=\"${namespace}\", job=~\"$job\",code=~\"4..\"}[$__rate_interval]))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Status 4xx", - "transformations": [], - "transparent": false, - "type": "stat" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": null, - "mappings": [], - "noValue": "0", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "semi-dark-blue", - "index": 0, - "line": true, - "op": "gt", - "value": "null", - "yaxis": "left" - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 5, - "y": 4 - }, - "hideTimeOverride": false, - "id": 10, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "background", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": null, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "index": 0, + "line": true, + "op": "gt", + "value": "null", + "yaxis": "left" + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 0 + }, + "hideTimeOverride": false, + "id": 2, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["mean"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": null, + "expr": "sum(rate(s3_cloudserver_http_requests_total{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Http requests rate", + "transformations": [], + "transparent": false, + "type": "stat" }, - "textMode": "auto" - }, - "targets": [ { - "datasource": null, - "expr": "sum(rate(s3_cloudserver_http_requests_total{namespace=\"${namespace}\", job=~\"$job\",code=~\"5..\"}[$__rate_interval]))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Status 5xx", - "transformations": [], - "transparent": false, - "type": "stat" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": null, - "mappings": [], - "noValue": "none", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "index": 0, - "line": true, - "op": "gt", - "value": "null", - "yaxis": "left" - }, - { - "color": "red", - "index": 1, - "line": true, - "op": "gt", - "value": 80.0, - "yaxis": "left" - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 7, - "y": 4 - }, - "hideTimeOverride": false, - "id": 11, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "calcs": ["mean"], + "decimals": null, + "limit": null, + "links": [], + "mappings": [], + "max": 100, + "min": 0, + "noValue": "-", + "override": {}, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#808080", + "index": 0, + "line": true, + "op": "gt", + "value": "null", + "yaxis": "left" + }, + { + "color": "red", + "index": 1, + "line": true, + "op": "gt", + "value": 0.0, + "yaxis": "left" + }, + { + "color": "orange", + "index": 2, + "line": true, + "op": "gt", + "value": 80.0, + "yaxis": "left" + }, + { + "color": "green", + "index": 3, + "line": true, + "op": "gt", + "value": 90.0, + "yaxis": "left" + } + ] + }, + "title": null, + "unit": "percent", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 6, + "y": 0 + }, + "hideTimeOverride": false, + "id": 3, + "links": [], + "maxDataPoints": 100, + "options": { + "reduceOptions": { + "calcs": ["mean"] + } + }, + "targets": [ + { + "datasource": null, + "expr": "sum(rate(s3_cloudserver_http_requests_total{namespace=\"${namespace}\", job=~\"$job\", code=~\"2..\"}[$__rate_interval])) * 100\n /\nsum(rate(s3_cloudserver_http_requests_total{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval]) > 0)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Success rate", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Success rate", + "transformations": [], + "transparent": false, + "type": "gauge" }, - "textMode": "auto" - }, - "targets": [ { - "datasource": null, - "expr": "sum(s3_cloudserver_http_active_requests{namespace=\"${namespace}\", job=~\"$job\"})", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Active requests", - "transformations": [], - "transparent": false, - "type": "stat" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Rate of data ingested out-of-band (OOB) : cumulative amount of OOB data created (>0) or deleted (<0) per second.", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": 1, - "mappings": [], - "noValue": "none", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "purple", - "index": 0, - "line": true, - "op": "gt", - "value": "null", - "yaxis": "left" - } - ] - }, - "unit": "binBps" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 9, - "y": 4 - }, - "hideTimeOverride": false, - "id": 12, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "background", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false + "datasource": "${DS_PROMETHEUS}", + "description": "Rate of data ingested : cumulative amount of data created (>0) or deleted (<0) per second.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 1, + "mappings": [], + "noValue": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-purple", + "index": 0, + "line": true, + "op": "gt", + "value": "null", + "yaxis": "left" + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 9, + "y": 0 + }, + "hideTimeOverride": false, + "id": 4, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["mean"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": null, + "expr": "-sum(deriv(s3_cloudserver_disk_available_bytes{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Injection Data Rate", + "transformations": [], + "transparent": false, + "type": "stat" }, - "textMode": "auto" - }, - "targets": [ { - "datasource": null, - "expr": "sum(deriv(s3_cloudserver_ingested_bytes{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval]))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "OOB Inject. Data Rate", - "transformations": [], - "transparent": false, - "type": "stat" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Rate of object ingested out-of-band (OOB) : cumulative count of OOB object created (>0) or deleted (<0) per second.", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": 1, - "mappings": [], - "noValue": "none", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "purple", - "index": 0, - "line": true, - "op": "gt", - "value": "null", - "yaxis": "left" - } - ] - }, - "unit": "O/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 12, - "y": 4 - }, - "hideTimeOverride": false, - "id": 13, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "background", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false + "datasource": "${DS_PROMETHEUS}", + "description": "Rate of object ingestion : cumulative count of object created (>0) or deleted (<0) per second.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 1, + "mappings": [], + "noValue": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-purple", + "index": 0, + "line": true, + "op": "gt", + "value": "null", + "yaxis": "left" + } + ] + }, + "unit": "O/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 0 + }, + "hideTimeOverride": false, + "id": 5, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["mean"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": null, + "expr": "sum(deriv(s3_cloudserver_objects_count{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Injection Rate", + "transformations": [], + "transparent": false, + "type": "stat" }, - "textMode": "auto" - }, - "targets": [ { - "datasource": null, - "expr": "sum(deriv(s3_cloudserver_ingested_objects_count{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval]))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "OOB Inject. Rate", - "transformations": [], - "transparent": false, - "type": "stat" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Number of S3 objects available in the cluster.\nThis value is computed asynchronously, and update may be delayed up to 1h.", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": null, - "mappings": [], - "noValue": "-", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#808080", - "index": 0, - "line": true, - "op": "gt", - "value": "null", - "yaxis": "left" - }, - { - "color": "blue", - "index": 1, - "line": true, - "op": "gt", - "value": 0.0, - "yaxis": "left" - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 7, - "x": 15, - "y": 4 - }, - "hideTimeOverride": false, - "id": 14, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + "datasource": "${DS_PROMETHEUS}", + "description": "Number of S3 buckets available in the cluster.\nThis value is computed asynchronously, and update may be delayed up to 1h.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": null, + "mappings": [], + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#808080", + "index": 0, + "line": true, + "op": "gt", + "value": "null", + "yaxis": "left" + }, + { + "color": "blue", + "index": 1, + "line": true, + "op": "gt", + "value": 0.0, + "yaxis": "left" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 7, + "x": 15, + "y": 0 + }, + "hideTimeOverride": false, + "id": 6, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": null, + "expr": "sum(s3_cloudserver_buckets_count{namespace=\"${namespace}\", job=\"${reportJob}\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Buckets", + "transformations": [], + "transparent": false, + "type": "stat" }, - "textMode": "auto" - }, - "targets": [ { - "datasource": null, - "expr": "sum(s3_cloudserver_objects_count{namespace=\"${namespace}\", job=\"${reportJob}\"})", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Objects", - "transformations": [], - "transparent": false, - "type": "stat" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Time elapsed since the last report, when object/bucket count was updated.", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": null, - "mappings": [], - "noValue": "-", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#808080", - "index": 0, - "line": true, - "op": "gt", - "value": "null", - "yaxis": "left" - }, - { - "color": "green", - "index": 1, - "line": true, - "op": "gt", - "value": 0.0, - "yaxis": "left" - }, - { - "color": "super-light-yellow", - "index": 2, - "line": true, - "op": "gt", - "value": 1800.0, - "yaxis": "left" - }, - { - "color": "orange", - "index": 3, - "line": true, - "op": "gt", - "value": 3600.0, - "yaxis": "left" - }, - { - "color": "red", - "index": 4, - "line": true, - "op": "gt", - "value": 3700.0, - "yaxis": "left" - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 22, - "y": 4 - }, - "hideTimeOverride": false, - "id": 15, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false + "datasource": "${DS_PROMETHEUS}", + "description": "Status of the reports-handler pod.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": null, + "mappings": [], + "noValue": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "index": 0, + "line": true, + "op": "gt", + "value": "null", + "yaxis": "left" + }, + { + "color": "green", + "index": 1, + "line": true, + "op": "gt", + "value": 1.0, + "yaxis": "left" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 22, + "y": 0 + }, + "hideTimeOverride": false, + "id": 7, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["last"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": null, + "expr": "sum(up{namespace=\"${namespace}\", job=\"${reportJob}\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Reporter", + "transformations": [], + "transparent": false, + "type": "stat" }, - "textMode": "auto" - }, - "targets": [ { - "datasource": null, - "expr": "time()\n- max(s3_cloudserver_last_report_timestamp{namespace=\"${namespace}\", job=\"${reportJob}\"})\n+ (max(s3_cloudserver_last_report_timestamp{namespace=\"${namespace}\", job=\"${reportJob}\"})\n - max(kube_cronjob_status_last_schedule_time{namespace=\"${namespace}\", cronjob=\"${countItemsJob}\"})\n > 0 or vector(0))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Last Report", - "transformations": [], - "transparent": false, - "type": "stat" - }, - { - "collapsed": false, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [] - } - } - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 8 - }, - "hideTimeOverride": false, - "id": 16, - "links": [], - "maxDataPoints": 100, - "panels": [], - "targets": [], - "title": "Response codes", - "transformations": [], - "transparent": false, - "type": "row" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 30, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "log": 2, - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": {}, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 9 - }, - "hideTimeOverride": false, - "id": 17, - "links": [], - "maxDataPoints": 100, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": null, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-blue", + "index": 0, + "line": true, + "op": "gt", + "value": "null", + "yaxis": "left" + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 4 + }, + "hideTimeOverride": false, + "id": 8, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["mean"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": null, + "expr": "sum(rate(s3_cloudserver_http_requests_total{namespace=\"${namespace}\", job=~\"$job\",code=\"200\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Status 200", + "transformations": [], + "transparent": false, + "type": "stat" }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ { - "datasource": null, - "expr": "sum by (code) (rate(s3_cloudserver_http_requests_total{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval]))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{code}}", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Http status code over time", - "transformations": [], - "transparent": false, - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 39, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "log": 2, - "type": "log" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [] - }, - "unit": "ops" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Success" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "dark-blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "User errors" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "semi-dark-orange", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "System errors" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "semi-dark-red", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 9 - }, - "hideTimeOverride": false, - "id": 18, - "links": [], - "maxDataPoints": 100, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": null, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-blue", + "index": 0, + "line": true, + "op": "gt", + "value": "null", + "yaxis": "left" + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 3, + "y": 4 + }, + "hideTimeOverride": false, + "id": 9, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["mean"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": null, + "expr": "sum(rate(s3_cloudserver_http_requests_total{namespace=\"${namespace}\", job=~\"$job\",code=~\"4..\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Status 4xx", + "transformations": [], + "transparent": false, + "type": "stat" }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ { - "datasource": null, - "expr": "sum(rate(s3_cloudserver_http_requests_total{namespace=\"${namespace}\", job=~\"$job\", code=~\"2..\"}[$__rate_interval]))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Success", - "metric": "", - "refId": "", - "step": 10, - "target": "" + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": null, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-blue", + "index": 0, + "line": true, + "op": "gt", + "value": "null", + "yaxis": "left" + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 5, + "y": 4 + }, + "hideTimeOverride": false, + "id": 10, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["mean"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": null, + "expr": "sum(rate(s3_cloudserver_http_requests_total{namespace=\"${namespace}\", job=~\"$job\",code=~\"5..\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Status 5xx", + "transformations": [], + "transparent": false, + "type": "stat" }, { - "datasource": null, - "expr": "sum(rate(s3_cloudserver_http_requests_total{namespace=\"${namespace}\", job=~\"$job\", code=~\"4..\"}[$__rate_interval]))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "User errors", - "metric": "", - "refId": "", - "step": 10, - "target": "" + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": null, + "mappings": [], + "noValue": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "index": 0, + "line": true, + "op": "gt", + "value": "null", + "yaxis": "left" + }, + { + "color": "red", + "index": 1, + "line": true, + "op": "gt", + "value": 80.0, + "yaxis": "left" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 7, + "y": 4 + }, + "hideTimeOverride": false, + "id": 11, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": null, + "expr": "sum(s3_cloudserver_http_active_requests{namespace=\"${namespace}\", job=~\"$job\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Active requests", + "transformations": [], + "transparent": false, + "type": "stat" }, { - "datasource": null, - "expr": "sum(rate(s3_cloudserver_http_requests_total{namespace=\"${namespace}\", job=~\"$job\", code=~\"5..\"}[$__rate_interval]))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "System errors", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Aggregated status over time", - "transformations": [], - "transparent": false, - "type": "timeseries" - }, - { - "collapsed": false, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [] - } - } - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 17 - }, - "hideTimeOverride": false, - "id": 19, - "links": [], - "maxDataPoints": 100, - "panels": [], - "targets": [], - "title": "Operations", - "transformations": [], - "transparent": false, - "type": "row" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "log": 2, - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": {}, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 18, - "x": 0, - "y": 18 - }, - "hideTimeOverride": false, - "id": 20, - "links": [], - "maxDataPoints": 100, - "options": { - "legend": { - "calcs": [ - "min", - "mean", - "max" - ], - "displayMode": "table", - "placement": "right" + "datasource": "${DS_PROMETHEUS}", + "description": "Rate of data ingested out-of-band (OOB) : cumulative amount of OOB data created (>0) or deleted (<0) per second.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 1, + "mappings": [], + "noValue": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "index": 0, + "line": true, + "op": "gt", + "value": "null", + "yaxis": "left" + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 9, + "y": 4 + }, + "hideTimeOverride": false, + "id": 12, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["mean"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": null, + "expr": "sum(deriv(s3_cloudserver_ingested_bytes{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "OOB Inject. Data Rate", + "transformations": [], + "transparent": false, + "type": "stat" }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ { - "datasource": null, - "expr": "sum(rate(s3_cloudserver_http_requests_total{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval])) by(action)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{action}}", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Request rate per S3 action", - "transformations": [], - "transparent": false, - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": {}, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 6, - "x": 18, - "y": 18 - }, - "hideTimeOverride": false, - "id": 21, - "links": [], - "maxDataPoints": 100, - "options": { - "displayLabels": [ - "name", - "percent" - ], - "legend": { - "displayMode": "list", - "placement": "bottom", - "values": [] - }, - "pieType": "pie", - "reduceOptions": { - "calcs": [ - "sum" - ], - "fields": "", - "values": false + "datasource": "${DS_PROMETHEUS}", + "description": "Rate of object ingested out-of-band (OOB) : cumulative count of OOB object created (>0) or deleted (<0) per second.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 1, + "mappings": [], + "noValue": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "index": 0, + "line": true, + "op": "gt", + "value": "null", + "yaxis": "left" + } + ] + }, + "unit": "O/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 4 + }, + "hideTimeOverride": false, + "id": 13, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["mean"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": null, + "expr": "sum(deriv(s3_cloudserver_ingested_objects_count{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "OOB Inject. Rate", + "transformations": [], + "transparent": false, + "type": "stat" }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ { - "datasource": null, - "expr": "sum(round(increase(s3_cloudserver_http_requests_total{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval]))) by(method)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{method}}", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "HTTP Method breakdown", - "transformations": [], - "transparent": false, - "type": "piechart" - }, - { - "collapsed": false, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [] - } - } - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 28 - }, - "hideTimeOverride": false, - "id": 22, - "links": [], - "maxDataPoints": 100, - "panels": [], - "targets": [], - "title": "Latency", - "transformations": [], - "transparent": false, - "type": "row" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "log": 2, - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": 180000, - "stacking": {}, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 29 - }, - "hideTimeOverride": false, - "id": 23, - "links": [], - "maxDataPoints": 100, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" + "datasource": "${DS_PROMETHEUS}", + "description": "Number of S3 objects available in the cluster.\nThis value is computed asynchronously, and update may be delayed up to 1h.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": null, + "mappings": [], + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#808080", + "index": 0, + "line": true, + "op": "gt", + "value": "null", + "yaxis": "left" + }, + { + "color": "blue", + "index": 1, + "line": true, + "op": "gt", + "value": 0.0, + "yaxis": "left" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 7, + "x": 15, + "y": 4 + }, + "hideTimeOverride": false, + "id": 14, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": null, + "expr": "sum(s3_cloudserver_objects_count{namespace=\"${namespace}\", job=\"${reportJob}\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Objects", + "transformations": [], + "transparent": false, + "type": "stat" }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ { - "datasource": null, - "expr": "sum(rate(s3_cloudserver_http_request_duration_seconds_sum{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval]))\n /\nsum(rate(s3_cloudserver_http_request_duration_seconds_count{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval]))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Overall", - "metric": "", - "refId": "", - "step": 10, - "target": "" + "datasource": "${DS_PROMETHEUS}", + "description": "Time elapsed since the last report, when object/bucket count was updated.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": null, + "mappings": [], + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#808080", + "index": 0, + "line": true, + "op": "gt", + "value": "null", + "yaxis": "left" + }, + { + "color": "green", + "index": 1, + "line": true, + "op": "gt", + "value": 0.0, + "yaxis": "left" + }, + { + "color": "super-light-yellow", + "index": 2, + "line": true, + "op": "gt", + "value": 1800.0, + "yaxis": "left" + }, + { + "color": "orange", + "index": 3, + "line": true, + "op": "gt", + "value": 3600.0, + "yaxis": "left" + }, + { + "color": "red", + "index": 4, + "line": true, + "op": "gt", + "value": 3700.0, + "yaxis": "left" + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 22, + "y": 4 + }, + "hideTimeOverride": false, + "id": 15, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["last"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": null, + "expr": "time()\n- max(s3_cloudserver_last_report_timestamp{namespace=\"${namespace}\", job=\"${reportJob}\"})\n+ (max(s3_cloudserver_last_report_timestamp{namespace=\"${namespace}\", job=\"${reportJob}\"})\n - max(kube_cronjob_status_last_schedule_time{namespace=\"${namespace}\", cronjob=\"${countItemsJob}\"})\n > 0 or vector(0))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Last Report", + "transformations": [], + "transparent": false, + "type": "stat" }, { - "datasource": null, - "expr": "sum(rate(s3_cloudserver_http_request_duration_seconds_sum{namespace=\"${namespace}\", job=~\"$job\", action=~\"objectPut|objectPutPart|objectCopy|objectPutCopyPart\"}[$__rate_interval]))\n /\nsum(rate(s3_cloudserver_http_request_duration_seconds_count{namespace=\"${namespace}\", job=~\"$job\", action=~\"objectPut|objectPutPart|objectCopy|objectPutCopyPart\"}[$__rate_interval]))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Upload", - "metric": "", - "refId": "", - "step": 10, - "target": "" + "collapsed": false, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [] + } + } + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "hideTimeOverride": false, + "id": 16, + "links": [], + "maxDataPoints": 100, + "panels": [], + "targets": [], + "title": "Response codes", + "transformations": [], + "transparent": false, + "type": "row" }, { - "datasource": null, - "expr": "sum(rate(s3_cloudserver_http_request_duration_seconds_sum{namespace=\"${namespace}\", job=~\"$job\", action=\"objectDelete\"}[$__rate_interval]))\n /\nsum(rate(s3_cloudserver_http_request_duration_seconds_count{namespace=\"${namespace}\", job=~\"$job\", action=\"objectDelete\"}[$__rate_interval]))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Delete", - "metric": "", - "refId": "", - "step": 10, - "target": "" + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": {}, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "hideTimeOverride": false, + "id": 17, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": null, + "expr": "sum by (code) (rate(s3_cloudserver_http_requests_total{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{code}}", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Http status code over time", + "transformations": [], + "transparent": false, + "type": "timeseries" }, { - "datasource": null, - "expr": "sum(rate(s3_cloudserver_http_request_duration_seconds_sum{namespace=\"${namespace}\", job=~\"$job\", action=\"objectGet\"}[$__rate_interval]))\n /\nsum(rate(s3_cloudserver_http_request_duration_seconds_count{namespace=\"${namespace}\", job=~\"$job\", action=\"objectGet\"}[$__rate_interval]))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Download", - "metric": "", - "refId": "", - "step": 10, - "target": "" + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 39, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "ops" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "User errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "System errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "hideTimeOverride": false, + "id": 18, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": null, + "expr": "sum(rate(s3_cloudserver_http_requests_total{namespace=\"${namespace}\", job=~\"$job\", code=~\"2..\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Success", + "metric": "", + "refId": "", + "step": 10, + "target": "" + }, + { + "datasource": null, + "expr": "sum(rate(s3_cloudserver_http_requests_total{namespace=\"${namespace}\", job=~\"$job\", code=~\"4..\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "User errors", + "metric": "", + "refId": "", + "step": 10, + "target": "" + }, + { + "datasource": null, + "expr": "sum(rate(s3_cloudserver_http_requests_total{namespace=\"${namespace}\", job=~\"$job\", code=~\"5..\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "System errors", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Aggregated status over time", + "transformations": [], + "transparent": false, + "type": "timeseries" }, { - "datasource": null, - "expr": "sum(rate(s3_cloudserver_http_request_duration_seconds_sum{namespace=\"${namespace}\", job=~\"$job\", action=~\"multiObjectDelete|multipartDelete\"}[$__rate_interval]))\n /\nsum(rate(s3_cloudserver_http_request_duration_seconds_count{namespace=\"${namespace}\", job=~\"$job\", action=~\"multiObjectDelete|multipartDelete\"}[$__rate_interval]))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Multi-delete", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Average latencies", - "transformations": [], - "transparent": false, - "type": "timeseries" - }, - { - "cards": { - "cardPadding": null, - "cardRound": null - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateOranges", - "exponent": 0.5, - "max": null, - "min": null, - "mode": "opacity" - }, - "dataFormat": "tsbuckets", - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [] - } - } - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 29 - }, - "heatmap": {}, - "hideTimeOverride": false, - "hideZeroBuckets": false, - "highlightCards": true, - "id": 24, - "legend": { - "show": false - }, - "links": [], - "maxDataPoints": 25, - "reverseYBuckets": false, - "targets": [ - { - "datasource": null, - "expr": "sum by(le) (increase(s3_cloudserver_http_request_duration_seconds_bucket{namespace=\"${namespace}\", job=\"${job}\"}[$__rate_interval]))", - "format": "heatmap", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ le }}", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Request duration", - "tooltip": { - "show": true, - "showHistogram": true - }, - "transformations": [], - "transparent": false, - "type": "heatmap", - "xAxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yAxis": { - "decimals": null, - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - }, - { - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "log": 2, - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": 180000, - "stacking": {}, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 37 - }, - "hideTimeOverride": false, - "id": 25, - "links": [], - "maxDataPoints": 100, - "options": { - "legend": { - "calcs": [ - "max", - "mean" - ], - "displayMode": "table", - "placement": "right" + "collapsed": false, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [] + } + } + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "hideTimeOverride": false, + "id": 19, + "links": [], + "maxDataPoints": 100, + "panels": [], + "targets": [], + "title": "Operations", + "transformations": [], + "transparent": false, + "type": "row" }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ { - "datasource": null, - "expr": "sum(rate(s3_cloudserver_http_request_duration_seconds_sum{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval])) by (action)\n /\nsum(rate(s3_cloudserver_http_request_duration_seconds_count{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval])) by (action)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{action}}", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Latencies per S3 action", - "transformations": [], - "transparent": false, - "type": "timeseries" - }, - { - "collapsed": false, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [] - } - } - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 47 - }, - "hideTimeOverride": false, - "id": 26, - "links": [], - "maxDataPoints": 100, - "panels": [], - "targets": [], - "title": "Data rate", - "transformations": [], - "transparent": false, - "type": "row" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "log": 2, - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": {}, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "index": 0, - "line": true, - "op": "gt", - "value": "null", - "yaxis": "left" - }, - { - "color": "red", - "index": 1, - "line": true, - "op": "gt", - "value": 80.0, - "yaxis": "left" - } - ] - }, - "unit": "binBps" + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": {}, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 18, + "x": 0, + "y": 18 + }, + "hideTimeOverride": false, + "id": 20, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": ["min", "mean", "max"], + "displayMode": "table", + "placement": "right" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": null, + "expr": "sum(rate(s3_cloudserver_http_requests_total{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval])) by(action)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{action}}", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Request rate per S3 action", + "transformations": [], + "transparent": false, + "type": "timeseries" }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Out" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 48 - }, - "hideTimeOverride": false, - "id": 27, - "links": [], - "maxDataPoints": 100, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" + { + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 6, + "x": 18, + "y": 18 + }, + "hideTimeOverride": false, + "id": 21, + "links": [], + "maxDataPoints": 100, + "options": { + "displayLabels": ["name", "percent"], + "legend": { + "displayMode": "list", + "placement": "bottom", + "values": [] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": ["sum"], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": null, + "expr": "sum(round(increase(s3_cloudserver_http_requests_total{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval]))) by(method)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{method}}", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "HTTP Method breakdown", + "transformations": [], + "transparent": false, + "type": "piechart" }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ { - "datasource": null, - "expr": "sum(rate(s3_cloudserver_http_response_size_bytes_sum{namespace=\"${namespace}\", job=\"${job}\"}[$__rate_interval]))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Out", - "metric": "", - "refId": "", - "step": 10, - "target": "" + "collapsed": false, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [] + } + } + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 28 + }, + "hideTimeOverride": false, + "id": 22, + "links": [], + "maxDataPoints": 100, + "panels": [], + "targets": [], + "title": "Latency", + "transformations": [], + "transparent": false, + "type": "row" }, { - "datasource": null, - "expr": "sum(rate(s3_cloudserver_http_request_size_bytes_sum{namespace=\"${namespace}\", job=\"${job}\"}[$__rate_interval]))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "In", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Bandwidth", - "transformations": [], - "transparent": false, - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#808080", - "index": 0, - "line": true, - "op": "gt", - "value": "null", - "yaxis": "left" - }, - { - "color": "green", - "index": 1, - "line": true, - "op": "gt", - "value": 0.0, - "yaxis": "left" - } - ] - } - } - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 12, - "y": 48 - }, - "hideTimeOverride": false, - "id": 28, - "links": [], - "maxDataPoints": 100, - "options": { - "displayMode": "gradient", - "fieldOptions": { - "calcs": [ - "lastNotNull" - ], - "defaults": { - "decimals": null, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 180000, + "stacking": {}, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 29 + }, + "hideTimeOverride": false, + "id": 23, "links": [], - "max": null, - "min": null, - "noValue": "-", - "title": null, - "unit": "bytes" - }, - "limit": null, - "mappings": [], - "override": {}, - "thresholds": [ - { - "color": "#808080", - "index": 0, - "line": true, - "op": "gt", - "value": "null", - "yaxis": "left" + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } }, - { - "color": "green", - "index": 1, - "line": true, - "op": "gt", - "value": 0.0, - "yaxis": "left" - } - ], - "values": false + "targets": [ + { + "datasource": null, + "expr": "sum(rate(s3_cloudserver_http_request_duration_seconds_sum{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval]))\n /\nsum(rate(s3_cloudserver_http_request_duration_seconds_count{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Overall", + "metric": "", + "refId": "", + "step": 10, + "target": "" + }, + { + "datasource": null, + "expr": "sum(rate(s3_cloudserver_http_request_duration_seconds_sum{namespace=\"${namespace}\", job=~\"$job\", action=~\"objectPut|objectPutPart|objectCopy|objectPutCopyPart\"}[$__rate_interval]))\n /\nsum(rate(s3_cloudserver_http_request_duration_seconds_count{namespace=\"${namespace}\", job=~\"$job\", action=~\"objectPut|objectPutPart|objectCopy|objectPutCopyPart\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Upload", + "metric": "", + "refId": "", + "step": 10, + "target": "" + }, + { + "datasource": null, + "expr": "sum(rate(s3_cloudserver_http_request_duration_seconds_sum{namespace=\"${namespace}\", job=~\"$job\", action=\"objectDelete\"}[$__rate_interval]))\n /\nsum(rate(s3_cloudserver_http_request_duration_seconds_count{namespace=\"${namespace}\", job=~\"$job\", action=\"objectDelete\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Delete", + "metric": "", + "refId": "", + "step": 10, + "target": "" + }, + { + "datasource": null, + "expr": "sum(rate(s3_cloudserver_http_request_duration_seconds_sum{namespace=\"${namespace}\", job=~\"$job\", action=\"objectGet\"}[$__rate_interval]))\n /\nsum(rate(s3_cloudserver_http_request_duration_seconds_count{namespace=\"${namespace}\", job=~\"$job\", action=\"objectGet\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Download", + "metric": "", + "refId": "", + "step": 10, + "target": "" + }, + { + "datasource": null, + "expr": "sum(rate(s3_cloudserver_http_request_duration_seconds_sum{namespace=\"${namespace}\", job=~\"$job\", action=~\"multiObjectDelete|multipartDelete\"}[$__rate_interval]))\n /\nsum(rate(s3_cloudserver_http_request_duration_seconds_count{namespace=\"${namespace}\", job=~\"$job\", action=~\"multiObjectDelete|multipartDelete\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Multi-delete", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Average latencies", + "transformations": [], + "transparent": false, + "type": "timeseries" }, - "orientation": "vertical", - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "targets": [ { - "datasource": null, - "expr": "avg(s3_cloudserver_http_request_size_bytes{namespace=\"${namespace}\", job=~\"$job\"}) by (quantile)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ quantile }}", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Avg upload chunk size by \u03c6-quantile", - "transformations": [], - "transparent": false, - "type": "bargauge" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#808080", - "index": 0, - "line": true, - "op": "gt", - "value": "null", - "yaxis": "left" - }, - { - "color": "green", - "index": 1, - "line": true, - "op": "gt", - "value": 0.0, - "yaxis": "left" - } - ] - } - } - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 18, - "y": 48 - }, - "hideTimeOverride": false, - "id": 29, - "links": [], - "maxDataPoints": 100, - "options": { - "displayMode": "gradient", - "fieldOptions": { - "calcs": [ - "lastNotNull" - ], - "defaults": { - "decimals": null, + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateOranges", + "exponent": 0.5, + "max": null, + "min": null, + "mode": "opacity" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 29 + }, + "heatmap": {}, + "hideTimeOverride": false, + "hideZeroBuckets": false, + "highlightCards": true, + "id": 24, + "legend": { + "show": false + }, "links": [], - "max": null, - "min": null, - "noValue": "-", - "title": null, - "unit": "bytes" - }, - "limit": null, - "mappings": [], - "override": {}, - "thresholds": [ - { - "color": "#808080", - "index": 0, - "line": true, - "op": "gt", - "value": "null", - "yaxis": "left" + "maxDataPoints": 25, + "reverseYBuckets": false, + "targets": [ + { + "datasource": null, + "expr": "sum by(le) (increase(s3_cloudserver_http_request_duration_seconds_bucket{namespace=\"${namespace}\", job=\"${job}\"}[$__rate_interval]))", + "format": "heatmap", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ le }}", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Request duration", + "tooltip": { + "show": true, + "showHistogram": true }, - { - "color": "green", - "index": 1, - "line": true, - "op": "gt", - "value": 0.0, - "yaxis": "left" + "transformations": [], + "transparent": false, + "type": "heatmap", + "xAxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yAxis": { + "decimals": null, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true } - ], - "values": false }, - "orientation": "vertical", - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "targets": [ { - "datasource": null, - "expr": "avg(s3_cloudserver_http_response_size_bytes{namespace=\"${namespace}\", job=~\"$job\"}) by (quantile)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ quantile }}", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Avg download chunk size by \u03c6-quantile", - "transformations": [], - "transparent": false, - "type": "bargauge" - }, - { - "collapsed": false, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [] - } - } - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 56 - }, - "hideTimeOverride": false, - "id": 30, - "links": [], - "maxDataPoints": 100, - "panels": [], - "targets": [], - "title": "Errors", - "transformations": [], - "transparent": false, - "type": "row" - }, - { - "datasource": "${DS_LOKI}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": {}, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 57 - }, - "hideTimeOverride": false, - "id": 31, - "links": [], - "maxDataPoints": 100, - "options": { - "displayLabels": [ - "name" - ], - "legend": { - "displayMode": "table", - "placement": "right", - "values": [ - "value" - ] - }, - "pieType": "donut", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 180000, + "stacking": {}, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 37 + }, + "hideTimeOverride": false, + "id": 25, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": ["max", "mean"], + "displayMode": "table", + "placement": "right" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": null, + "expr": "sum(rate(s3_cloudserver_http_request_duration_seconds_sum{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval])) by (action)\n /\nsum(rate(s3_cloudserver_http_request_duration_seconds_count{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval])) by (action)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{action}}", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Latencies per S3 action", + "transformations": [], + "transparent": false, + "type": "timeseries" }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ { - "datasource": null, - "expr": "topk(10, sum by(bucketName) (\n count_over_time({namespace=\"${namespace}\", pod=~\"$pod\"}\n | json | bucketName!=\"\" and httpCode=\"404\"\n [$__interval])\n))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{bucketName}}", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "404 : Top10 by Bucket", - "transformations": [], - "transparent": false, - "type": "piechart" - }, - { - "datasource": "${DS_LOKI}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": {}, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 57 - }, - "hideTimeOverride": false, - "id": 32, - "links": [], - "maxDataPoints": 100, - "options": { - "displayLabels": [ - "name" - ], - "legend": { - "displayMode": "table", - "placement": "right", - "values": [ - "value" - ] - }, - "pieType": "donut", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + "collapsed": false, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [] + } + } + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 47 + }, + "hideTimeOverride": false, + "id": 26, + "links": [], + "maxDataPoints": 100, + "panels": [], + "targets": [], + "title": "Data rate", + "transformations": [], + "transparent": false, + "type": "row" }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ { - "datasource": null, - "expr": "topk(10, sum by(bucketName) (\n count_over_time({namespace=\"${namespace}\", pod=~\"$pod\"}\n | json | bucketName!=\"\" and httpCode=\"500\"\n [$__interval])\n))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{bucketName}}", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "500 : Top10 by Bucket", - "transformations": [], - "transparent": false, - "type": "piechart" - }, - { - "datasource": "${DS_LOKI}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": {}, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 57 - }, - "hideTimeOverride": false, - "id": 33, - "links": [], - "maxDataPoints": 100, - "options": { - "displayLabels": [ - "name" - ], - "legend": { - "displayMode": "table", - "placement": "right", - "values": [ - "value" - ] - }, - "pieType": "donut", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": {}, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "index": 0, + "line": true, + "op": "gt", + "value": "null", + "yaxis": "left" + }, + { + "color": "red", + "index": 1, + "line": true, + "op": "gt", + "value": 80.0, + "yaxis": "left" + } + ] + }, + "unit": "binBps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Out" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 48 + }, + "hideTimeOverride": false, + "id": 27, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": null, + "expr": "sum(rate(s3_cloudserver_http_response_size_bytes_sum{namespace=\"${namespace}\", job=\"${job}\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Out", + "metric": "", + "refId": "", + "step": 10, + "target": "" + }, + { + "datasource": null, + "expr": "sum(rate(s3_cloudserver_http_request_size_bytes_sum{namespace=\"${namespace}\", job=\"${job}\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "In", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Bandwidth", + "transformations": [], + "transparent": false, + "type": "timeseries" }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ { - "datasource": null, - "expr": "topk(10, sum by(bucketName) (\n count_over_time({namespace=\"${namespace}\", pod=~\"$pod\"}\n | json | bucketName!=\"\" and httpCode=~\"5..\"\n [$__interval])\n))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{bucketName}}", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "5xx : Top10 by Bucket", - "transformations": [], - "transparent": false, - "type": "piechart" - }, - { - "collapsed": false, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [] - } - } - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 65 - }, - "hideTimeOverride": false, - "id": 34, - "links": [], - "maxDataPoints": 100, - "panels": [], - "targets": [], - "title": "Quotas", - "transformations": [], - "transparent": false, - "type": "row" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Number of S3 buckets with quota enabled in the cluster.\nThis value is computed asynchronously, and update may be delayed up to 1h.", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": null, - "mappings": [], - "noValue": "-", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#808080", - "index": 0, - "line": true, - "op": "gt", - "value": "null", - "yaxis": "left" - }, - { - "color": "blue", - "index": 1, - "line": true, - "op": "gt", - "value": 0.0, - "yaxis": "left" - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 66 - }, - "hideTimeOverride": false, - "id": 35, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#808080", + "index": 0, + "line": true, + "op": "gt", + "value": "null", + "yaxis": "left" + }, + { + "color": "green", + "index": 1, + "line": true, + "op": "gt", + "value": 0.0, + "yaxis": "left" + } + ] + } + } + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 48 + }, + "hideTimeOverride": false, + "id": 28, + "links": [], + "maxDataPoints": 100, + "options": { + "displayMode": "gradient", + "fieldOptions": { + "calcs": ["lastNotNull"], + "defaults": { + "decimals": null, + "links": [], + "max": null, + "min": null, + "noValue": "-", + "title": null, + "unit": "bytes" + }, + "limit": null, + "mappings": [], + "override": {}, + "thresholds": [ + { + "color": "#808080", + "index": 0, + "line": true, + "op": "gt", + "value": "null", + "yaxis": "left" + }, + { + "color": "green", + "index": 1, + "line": true, + "op": "gt", + "value": 0.0, + "yaxis": "left" + } + ], + "values": false + }, + "orientation": "vertical", + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "targets": [ + { + "datasource": null, + "expr": "avg(s3_cloudserver_http_request_size_bytes{namespace=\"${namespace}\", job=~\"$job\"}) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ quantile }}", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Avg upload chunk size by \u03c6-quantile", + "transformations": [], + "transparent": false, + "type": "bargauge" }, - "textMode": "auto" - }, - "targets": [ { - "datasource": null, - "expr": "max(s3_cloudserver_quota_buckets_count{namespace=\"${namespace}\", job=~\"${reportJob}\"})", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Buckets with quota", - "transformations": [], - "transparent": false, - "type": "stat" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Number of accounts with quota enabled in the cluster.\nThis value is computed asynchronously, and update may be delayed up to 1h.", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": null, - "mappings": [], - "noValue": "-", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#808080", - "index": 0, - "line": true, - "op": "gt", - "value": "null", - "yaxis": "left" - }, - { - "color": "blue", - "index": 1, - "line": true, - "op": "gt", - "value": 0.0, - "yaxis": "left" - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 70 - }, - "hideTimeOverride": false, - "id": 36, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#808080", + "index": 0, + "line": true, + "op": "gt", + "value": "null", + "yaxis": "left" + }, + { + "color": "green", + "index": 1, + "line": true, + "op": "gt", + "value": 0.0, + "yaxis": "left" + } + ] + } + } + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 48 + }, + "hideTimeOverride": false, + "id": 29, + "links": [], + "maxDataPoints": 100, + "options": { + "displayMode": "gradient", + "fieldOptions": { + "calcs": ["lastNotNull"], + "defaults": { + "decimals": null, + "links": [], + "max": null, + "min": null, + "noValue": "-", + "title": null, + "unit": "bytes" + }, + "limit": null, + "mappings": [], + "override": {}, + "thresholds": [ + { + "color": "#808080", + "index": 0, + "line": true, + "op": "gt", + "value": "null", + "yaxis": "left" + }, + { + "color": "green", + "index": 1, + "line": true, + "op": "gt", + "value": 0.0, + "yaxis": "left" + } + ], + "values": false + }, + "orientation": "vertical", + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "targets": [ + { + "datasource": null, + "expr": "avg(s3_cloudserver_http_response_size_bytes{namespace=\"${namespace}\", job=~\"$job\"}) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ quantile }}", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Avg download chunk size by \u03c6-quantile", + "transformations": [], + "transparent": false, + "type": "bargauge" }, - "textMode": "auto" - }, - "targets": [ { - "datasource": null, - "expr": "max(s3_cloudserver_quota_accounts_count{namespace=\"${namespace}\", job=~\"${reportJob}\"})", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Accounts with quota", - "transformations": [], - "transparent": false, - "type": "stat" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 30, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "log": 2, - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": {}, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 6, - "y": 66 - }, - "hideTimeOverride": false, - "id": 37, - "links": [], - "maxDataPoints": 100, - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" + "collapsed": false, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [] + } + } + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 56 + }, + "hideTimeOverride": false, + "id": 30, + "links": [], + "maxDataPoints": 100, + "panels": [], + "targets": [], + "title": "Errors", + "transformations": [], + "transparent": false, + "type": "row" }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ { - "datasource": null, - "expr": "sum(rate(s3_cloudserver_quota_unavailable_count{namespace=\"${namespace}\", job=\"${job}\"}[$__rate_interval]))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Operations with unavailable metrics", - "transformations": [], - "transparent": false, - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "log": 2, - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": {}, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 66 - }, - "hideTimeOverride": false, - "id": 38, - "links": [], - "maxDataPoints": 100, - "options": { - "legend": { - "calcs": [ - "min", - "mean", - "max" - ], - "displayMode": "table", - "placement": "right" + "datasource": "${DS_LOKI}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 57 + }, + "hideTimeOverride": false, + "id": 31, + "links": [], + "maxDataPoints": 100, + "options": { + "displayLabels": ["name"], + "legend": { + "displayMode": "table", + "placement": "right", + "values": ["value"] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": null, + "expr": "topk(10, sum by(bucketName) (\n count_over_time({namespace=\"${namespace}\", pod=~\"$pod\"}\n | json | bucketName!=\"\" and httpCode=\"404\"\n [$__interval])\n))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{bucketName}}", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "404 : Top10 by Bucket", + "transformations": [], + "transparent": false, + "type": "piechart" }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ { - "datasource": null, - "expr": "sum(rate(s3_cloudserver_quota_evaluation_duration_seconds_count{namespace=\"${namespace}\", job=\"${job}\"}[$__rate_interval])) by(action)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{action}}", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Quota evaluaton rate per S3 action", - "transformations": [], - "transparent": false, - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 30, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "stepAfter", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "log": 2, - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": {}, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "index": 0, - "line": true, - "op": "gt", - "value": "null", - "yaxis": "left" - }, - { - "color": "orange", - "index": 1, - "line": true, - "op": "gt", - "value": 90.0, - "yaxis": "left" - }, - { - "color": "red", - "index": 2, - "line": true, - "op": "gt", - "value": 0.0, - "yaxis": "left" - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 0, - "y": 74 - }, - "hideTimeOverride": false, - "id": 39, - "links": [], - "maxDataPoints": 100, - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" + "datasource": "${DS_LOKI}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 57 + }, + "hideTimeOverride": false, + "id": 32, + "links": [], + "maxDataPoints": 100, + "options": { + "displayLabels": ["name"], + "legend": { + "displayMode": "table", + "placement": "right", + "values": ["value"] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": null, + "expr": "topk(10, sum by(bucketName) (\n count_over_time({namespace=\"${namespace}\", pod=~\"$pod\"}\n | json | bucketName!=\"\" and httpCode=\"500\"\n [$__interval])\n))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{bucketName}}", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "500 : Top10 by Bucket", + "transformations": [], + "transparent": false, + "type": "piechart" }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ { - "datasource": null, - "expr": "avg(avg_over_time(s3_cloudserver_quota_utilization_service_available{namespace=\"${namespace}\",job=\"${job}\"}[$__rate_interval])) * 100", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Quota service uptime", - "transformations": [], - "transparent": false, - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 30, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "log": 2, - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": {}, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [] - }, - "unit": "ops" + "datasource": "${DS_LOKI}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 57 + }, + "hideTimeOverride": false, + "id": 33, + "links": [], + "maxDataPoints": 100, + "options": { + "displayLabels": ["name"], + "legend": { + "displayMode": "table", + "placement": "right", + "values": ["value"] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": null, + "expr": "topk(10, sum by(bucketName) (\n count_over_time({namespace=\"${namespace}\", pod=~\"$pod\"}\n | json | bucketName!=\"\" and httpCode=~\"5..\"\n [$__interval])\n))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{bucketName}}", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "5xx : Top10 by Bucket", + "transformations": [], + "transparent": false, + "type": "piechart" }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 6, - "y": 74 - }, - "hideTimeOverride": false, - "id": 40, - "links": [], - "maxDataPoints": 100, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" + { + "collapsed": false, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [] + } + } + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 65 + }, + "hideTimeOverride": false, + "id": 34, + "links": [], + "maxDataPoints": 100, + "panels": [], + "targets": [], + "title": "Quotas", + "transformations": [], + "transparent": false, + "type": "row" }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ { - "datasource": null, - "expr": "sum(rate(s3_cloudserver_quota_evaluation_duration_seconds_count{namespace=\"${namespace}\", code=~\"2..\", job=\"${job}\"}[$__rate_interval]))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Success", - "metric": "", - "refId": "", - "step": 10, - "target": "" + "datasource": "${DS_PROMETHEUS}", + "description": "Number of S3 buckets with quota enabled in the cluster.\nThis value is computed asynchronously, and update may be delayed up to 1h.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": null, + "mappings": [], + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#808080", + "index": 0, + "line": true, + "op": "gt", + "value": "null", + "yaxis": "left" + }, + { + "color": "blue", + "index": 1, + "line": true, + "op": "gt", + "value": 0.0, + "yaxis": "left" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 66 + }, + "hideTimeOverride": false, + "id": 35, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": null, + "expr": "max(s3_cloudserver_quota_buckets_count{namespace=\"${namespace}\", job=~\"${reportJob}\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Buckets with quota", + "transformations": [], + "transparent": false, + "type": "stat" }, { - "datasource": null, - "expr": "sum(rate(s3_cloudserver_quota_evaluation_duration_seconds_count{namespace=\"${namespace}\", code=\"429\", job=\"${job}\"}[$__rate_interval]))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Quota Exceeded", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Quota evaluation status code over time", - "transformations": [], - "transparent": false, - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "log": 2, - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": 180000, - "stacking": {}, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [] - }, - "unit": "s" + "datasource": "${DS_PROMETHEUS}", + "description": "Number of accounts with quota enabled in the cluster.\nThis value is computed asynchronously, and update may be delayed up to 1h.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": null, + "mappings": [], + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#808080", + "index": 0, + "line": true, + "op": "gt", + "value": "null", + "yaxis": "left" + }, + { + "color": "blue", + "index": 1, + "line": true, + "op": "gt", + "value": 0.0, + "yaxis": "left" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 70 + }, + "hideTimeOverride": false, + "id": 36, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": null, + "expr": "max(s3_cloudserver_quota_accounts_count{namespace=\"${namespace}\", job=~\"${reportJob}\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Accounts with quota", + "transformations": [], + "transparent": false, + "type": "stat" }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 74 - }, - "hideTimeOverride": false, - "id": 41, - "links": [], - "maxDataPoints": 100, - "options": { - "legend": { - "calcs": [ - "min", - "mean", - "max" - ], - "displayMode": "table", - "placement": "right" + { + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": {}, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 66 + }, + "hideTimeOverride": false, + "id": 37, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": null, + "expr": "sum(rate(s3_cloudserver_quota_unavailable_count{namespace=\"${namespace}\", job=\"${job}\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Operations with unavailable metrics", + "transformations": [], + "transparent": false, + "type": "timeseries" }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ { - "datasource": null, - "expr": "sum(rate(s3_cloudserver_quota_evaluation_duration_seconds_sum{namespace=\"${namespace}\", job=\"${job}\", code=~\"2..\"}[$__rate_interval])) by (type)\n /\nsum(rate(s3_cloudserver_quota_evaluation_duration_seconds_count{namespace=\"${namespace}\", job=\"${job}\", code=~\"2..\"}[$__rate_interval])) by (type)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ type }} (success)", - "metric": "", - "refId": "", - "step": 10, - "target": "" + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": {}, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 66 + }, + "hideTimeOverride": false, + "id": 38, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": ["min", "mean", "max"], + "displayMode": "table", + "placement": "right" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": null, + "expr": "sum(rate(s3_cloudserver_quota_evaluation_duration_seconds_count{namespace=\"${namespace}\", job=\"${job}\"}[$__rate_interval])) by(action)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{action}}", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Quota evaluaton rate per S3 action", + "transformations": [], + "transparent": false, + "type": "timeseries" }, { - "datasource": null, - "expr": "sum(rate(s3_cloudserver_quota_evaluation_duration_seconds_sum{namespace=\"${namespace}\", job=\"${job}\", code=\"429\"}[$__rate_interval])) by (type)\n /\nsum(rate(s3_cloudserver_quota_evaluation_duration_seconds_count{namespace=\"${namespace}\", job=\"${job}\", code=\"429\"}[$__rate_interval])) by (type)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ type }} (exceeded)", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Average quota evaluation latencies", - "transformations": [], - "transparent": false, - "type": "timeseries" - }, - { - "cards": { - "cardPadding": null, - "cardRound": null - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateOranges", - "exponent": 0.5, - "max": null, - "min": null, - "mode": "opacity" - }, - "dataFormat": "tsbuckets", - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [] - } - } - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 0, - "y": 82 - }, - "heatmap": {}, - "hideTimeOverride": false, - "hideZeroBuckets": false, - "highlightCards": true, - "id": 42, - "legend": { - "show": false - }, - "links": [], - "maxDataPoints": 25, - "reverseYBuckets": false, - "targets": [ + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": {}, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "index": 0, + "line": true, + "op": "gt", + "value": "null", + "yaxis": "left" + }, + { + "color": "orange", + "index": 1, + "line": true, + "op": "gt", + "value": 90.0, + "yaxis": "left" + }, + { + "color": "red", + "index": 2, + "line": true, + "op": "gt", + "value": 0.0, + "yaxis": "left" + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 74 + }, + "hideTimeOverride": false, + "id": 39, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": null, + "expr": "avg(avg_over_time(s3_cloudserver_quota_utilization_service_available{namespace=\"${namespace}\",job=\"${job}\"}[$__rate_interval])) * 100", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Quota service uptime", + "transformations": [], + "transparent": false, + "type": "timeseries" + }, { - "datasource": null, - "expr": "sum by(le) (increase(s3_cloudserver_quota_evaluation_duration_seconds_bucket{namespace=\"${namespace}\", job=\"${job}\"}[$__rate_interval]))", - "format": "heatmap", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ le }}", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Quota evaluation duration", - "tooltip": { - "show": true, - "showHistogram": true - }, - "transformations": [], - "transparent": false, - "type": "heatmap", - "xAxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yAxis": { - "decimals": null, - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - }, - { - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "log": 2, - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": 180000, - "stacking": {}, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [] - }, - "unit": "s" + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": {}, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 74 + }, + "hideTimeOverride": false, + "id": 40, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": null, + "expr": "sum(rate(s3_cloudserver_quota_evaluation_duration_seconds_count{namespace=\"${namespace}\", code=~\"2..\", job=\"${job}\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Success", + "metric": "", + "refId": "", + "step": 10, + "target": "" + }, + { + "datasource": null, + "expr": "sum(rate(s3_cloudserver_quota_evaluation_duration_seconds_count{namespace=\"${namespace}\", code=\"429\", job=\"${job}\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Quota Exceeded", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Quota evaluation status code over time", + "transformations": [], + "transparent": false, + "type": "timeseries" }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 18, - "x": 6, - "y": 82 - }, - "hideTimeOverride": false, - "id": 43, - "links": [], - "maxDataPoints": 100, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" + { + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 180000, + "stacking": {}, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 74 + }, + "hideTimeOverride": false, + "id": 41, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": ["min", "mean", "max"], + "displayMode": "table", + "placement": "right" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": null, + "expr": "sum(rate(s3_cloudserver_quota_evaluation_duration_seconds_sum{namespace=\"${namespace}\", job=\"${job}\", code=~\"2..\"}[$__rate_interval])) by (type)\n /\nsum(rate(s3_cloudserver_quota_evaluation_duration_seconds_count{namespace=\"${namespace}\", job=\"${job}\", code=~\"2..\"}[$__rate_interval])) by (type)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ type }} (success)", + "metric": "", + "refId": "", + "step": 10, + "target": "" + }, + { + "datasource": null, + "expr": "sum(rate(s3_cloudserver_quota_evaluation_duration_seconds_sum{namespace=\"${namespace}\", job=\"${job}\", code=\"429\"}[$__rate_interval])) by (type)\n /\nsum(rate(s3_cloudserver_quota_evaluation_duration_seconds_count{namespace=\"${namespace}\", job=\"${job}\", code=\"429\"}[$__rate_interval])) by (type)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ type }} (exceeded)", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Average quota evaluation latencies", + "transformations": [], + "transparent": false, + "type": "timeseries" }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ { - "datasource": null, - "expr": "sum(rate(s3_cloudserver_quota_metrics_retrieval_duration_seconds_sum{namespace=\"${namespace}\", job=\"${job}\", code=~\"2..\"}[$__rate_interval])) by (class)\n /\nsum(rate(s3_cloudserver_quota_metrics_retrieval_duration_seconds_count{namespace=\"${namespace}\", job=\"${job}\", code=~\"2..\"}[$__rate_interval])) by (class)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ class }} (success)", - "metric": "", - "refId": "", - "step": 10, - "target": "" + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateOranges", + "exponent": 0.5, + "max": null, + "min": null, + "mode": "opacity" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 82 + }, + "heatmap": {}, + "hideTimeOverride": false, + "hideZeroBuckets": false, + "highlightCards": true, + "id": 42, + "legend": { + "show": false + }, + "links": [], + "maxDataPoints": 25, + "reverseYBuckets": false, + "targets": [ + { + "datasource": null, + "expr": "sum by(le) (increase(s3_cloudserver_quota_evaluation_duration_seconds_bucket{namespace=\"${namespace}\", job=\"${job}\"}[$__rate_interval]))", + "format": "heatmap", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ le }}", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Quota evaluation duration", + "tooltip": { + "show": true, + "showHistogram": true + }, + "transformations": [], + "transparent": false, + "type": "heatmap", + "xAxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yAxis": { + "decimals": null, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } }, { - "datasource": null, - "expr": "sum(rate(s3_cloudserver_quota_metrics_retrieval_duration_seconds_sum{namespace=\"${namespace}\", job=\"${job}\", code=~\"4..|5..\"}[$__rate_interval])) by (class)\n /\nsum(rate(s3_cloudserver_quota_metrics_retrieval_duration_seconds_count{namespace=\"${namespace}\", job=\"${job}\", code=~\"4..|5..\"}[$__rate_interval])) by (class)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ class }} (error)", - "metric": "", - "refId": "", - "step": 10, - "target": "" + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 180000, + "stacking": {}, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 18, + "x": 6, + "y": 82 + }, + "hideTimeOverride": false, + "id": 43, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": null, + "expr": "sum(rate(s3_cloudserver_quota_metrics_retrieval_duration_seconds_sum{namespace=\"${namespace}\", job=\"${job}\", code=~\"2..\"}[$__rate_interval])) by (class)\n /\nsum(rate(s3_cloudserver_quota_metrics_retrieval_duration_seconds_count{namespace=\"${namespace}\", job=\"${job}\", code=~\"2..\"}[$__rate_interval])) by (class)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ class }} (success)", + "metric": "", + "refId": "", + "step": 10, + "target": "" + }, + { + "datasource": null, + "expr": "sum(rate(s3_cloudserver_quota_metrics_retrieval_duration_seconds_sum{namespace=\"${namespace}\", job=\"${job}\", code=~\"4..|5..\"}[$__rate_interval])) by (class)\n /\nsum(rate(s3_cloudserver_quota_metrics_retrieval_duration_seconds_count{namespace=\"${namespace}\", job=\"${job}\", code=~\"4..|5..\"}[$__rate_interval])) by (class)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ class }} (error)", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Average utilization metrics retrieval latencies", + "transformations": [], + "transparent": false, + "type": "timeseries" } - ], - "title": "Average utilization metrics retrieval latencies", - "transformations": [], - "transparent": false, - "type": "timeseries" - } - ], - "refresh": "30s", - "rows": [], - "schemaVersion": 12, - "sharedCrosshair": false, - "style": "dark", - "tags": [ - "CloudServer" - ], - "templating": { - "list": [ - { - "allValue": null, - "auto": false, - "auto_count": 30, - "auto_min": "10s", - "current": { - "selected": false, - "tags": [], - "text": null, - "value": null - }, - "datasource": "${DS_PROMETHEUS}", - "hide": 0, - "includeAll": false, - "label": "Group", - "multi": true, - "name": "job", - "options": [], - "query": "label_values(s3_cloudserver_http_active_requests{namespace=\"${namespace}\", container=\"${container}\"}, job)", - "refresh": 1, - "regex": "/(?${zenkoName}-(?\\w*).*)/", - "sort": 1, - "tagValuesQuery": null, - "tagsQuery": null, - "type": "query", - "useTags": false - }, - { - "allValue": null, - "auto": false, - "auto_count": 30, - "auto_min": "10s", - "current": { - "selected": false, - "tags": [], - "text": null, - "value": null - }, - "datasource": "${DS_PROMETHEUS}", - "hide": 2, - "includeAll": false, - "label": "pod", - "multi": false, - "name": "pod", - "options": [], - "query": "label_values(s3_cloudserver_http_active_requests{namespace=\"${namespace}\", container=\"${container}\", job=~\"$job\"}, pod)", - "refresh": 1, - "regex": null, - "sort": 1, - "tagValuesQuery": null, - "tagsQuery": null, - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "hidden": false, - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "S3 service", - "uid": null, - "version": 110 + "refresh": "30s", + "rows": [], + "schemaVersion": 12, + "sharedCrosshair": false, + "style": "dark", + "tags": ["CloudServer"], + "templating": { + "list": [ + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "selected": false, + "tags": [], + "text": null, + "value": null + }, + "datasource": "${DS_PROMETHEUS}", + "hide": 0, + "includeAll": false, + "label": "Group", + "multi": true, + "name": "job", + "options": [], + "query": "label_values(s3_cloudserver_http_active_requests{namespace=\"${namespace}\", container=\"${container}\"}, job)", + "refresh": 1, + "regex": "/(?${zenkoName}-(?\\w*).*)/", + "sort": 1, + "tagValuesQuery": null, + "tagsQuery": null, + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "selected": false, + "tags": [], + "text": null, + "value": null + }, + "datasource": "${DS_PROMETHEUS}", + "hide": 2, + "includeAll": false, + "label": "pod", + "multi": false, + "name": "pod", + "options": [], + "query": "label_values(s3_cloudserver_http_active_requests{namespace=\"${namespace}\", container=\"${container}\", job=~\"$job\"}, pod)", + "refresh": 1, + "regex": null, + "sort": 1, + "tagValuesQuery": null, + "tagsQuery": null, + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "hidden": false, + "refresh_intervals": ["5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d"], + "time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d"] + }, + "timezone": "", + "title": "S3 service", + "uid": null, + "version": 110 } diff --git a/tests/functional/aws-node-sdk/lib/fixtures/project.js b/tests/functional/aws-node-sdk/lib/fixtures/project.js index 3b71a4ff79..1c1a4bbdda 100644 --- a/tests/functional/aws-node-sdk/lib/fixtures/project.js +++ b/tests/functional/aws-node-sdk/lib/fixtures/project.js @@ -10,9 +10,7 @@ const fakeDataSource = { generateManyBucketNames(numberOfBuckets) { const random = Math.round(Math.random() * 100).toString(); - return Array - .from(Array(numberOfBuckets).keys()) - .map(i => `${baseName}-${random}-${i}`); + return Array.from(Array(numberOfBuckets).keys()).map(i => `${baseName}-${random}-${i}`); }, }; diff --git a/tests/functional/aws-node-sdk/lib/json/mem_credentials.json b/tests/functional/aws-node-sdk/lib/json/mem_credentials.json index 66c4599b8e..b55449f72e 100644 --- a/tests/functional/aws-node-sdk/lib/json/mem_credentials.json +++ b/tests/functional/aws-node-sdk/lib/json/mem_credentials.json @@ -1,18 +1,18 @@ { - "default": { - "accessKey": "accessKey1", - "secretKey": "verySecretKey1" - }, - "lisa": { - "accessKey": "accessKey2", - "secretKey": "verySecretKey2" - }, - "replication": { - "accessKey": "replicationKey1", - "secretKey": "replicationSecretKey1" - }, - "vault": { - "accessKey": "TESTAK00000000000000", - "secretKey": "TESTSK0000000000000000000000000000000000" - } + "default": { + "accessKey": "accessKey1", + "secretKey": "verySecretKey1" + }, + "lisa": { + "accessKey": "accessKey2", + "secretKey": "verySecretKey2" + }, + "replication": { + "accessKey": "replicationKey1", + "secretKey": "replicationSecretKey1" + }, + "vault": { + "accessKey": "TESTAK00000000000000", + "secretKey": "TESTSK0000000000000000000000000000000000" + } } diff --git a/tests/functional/aws-node-sdk/lib/json/s3c_credentials.json b/tests/functional/aws-node-sdk/lib/json/s3c_credentials.json index af23d0e8cc..75f09cc09f 100644 --- a/tests/functional/aws-node-sdk/lib/json/s3c_credentials.json +++ b/tests/functional/aws-node-sdk/lib/json/s3c_credentials.json @@ -1,14 +1,14 @@ { - "default": { - "accessKey": "ACC1AK00000000000000", - "secretKey": "ACC1SK0000000000000000000000000000000000" - }, - "lisa": { - "accessKey": "ACC2AK00000000000000", - "secretKey": "ACC2SK0000000000000000000000000000000000" - }, - "replication": { - "accessKey": "ACCREPAK000000000000", - "secretKey": "ACCREPSK00000000000000000000000000000000" - } + "default": { + "accessKey": "ACC1AK00000000000000", + "secretKey": "ACC1SK0000000000000000000000000000000000" + }, + "lisa": { + "accessKey": "ACC2AK00000000000000", + "secretKey": "ACC2SK0000000000000000000000000000000000" + }, + "replication": { + "accessKey": "ACCREPAK000000000000", + "secretKey": "ACCREPSK00000000000000000000000000000000" + } } diff --git a/tests/functional/aws-node-sdk/lib/utility/bucket-util.js b/tests/functional/aws-node-sdk/lib/utility/bucket-util.js index f214540e30..aaa05bf483 100644 --- a/tests/functional/aws-node-sdk/lib/utility/bucket-util.js +++ b/tests/functional/aws-node-sdk/lib/utility/bucket-util.js @@ -21,18 +21,18 @@ class BucketUtility { credentials: { accessKeyId: '', secretAccessKey: '' }, forcePathStyle: true, signer: { sign: async request => request }, - }); - } - else { + }); + } else { this.s3 = new S3Client({ ...s3Config, maxAttempts: 0, - }); - } + }); + } } bucketExists(bucketName) { - return this.s3.send(new HeadBucketCommand({ Bucket: bucketName })) + return this.s3 + .send(new HeadBucketCommand({ Bucket: bucketName })) .then(() => true) .catch(err => { if (err.name === 'NotFound') { @@ -43,7 +43,8 @@ class BucketUtility { } createOne(bucketName) { - return this.s3.send(new CreateBucketCommand({ Bucket: bucketName })) + return this.s3 + .send(new CreateBucketCommand({ Bucket: bucketName })) .then(() => bucketName) .catch(err => { throw err; @@ -51,16 +52,18 @@ class BucketUtility { } createOneWithLock(bucketName) { - return this.s3.send(new CreateBucketCommand({ - Bucket: bucketName, - ObjectLockEnabledForBucket: true, - })).then(() => bucketName); + return this.s3 + .send( + new CreateBucketCommand({ + Bucket: bucketName, + ObjectLockEnabledForBucket: true, + }), + ) + .then(() => bucketName); } createMany(bucketNames) { - const promises = bucketNames.map(bucketName => - this.createOne(bucketName), - ); + const promises = bucketNames.map(bucketName => this.createOne(bucketName)); return Promise.all(promises); } @@ -69,9 +72,7 @@ class BucketUtility { const bucketName = projectFixture.generateBucketName(); return this.createOne(bucketName); } - const bucketNames = projectFixture - .generateManyBucketNames(nBuckets) - .sort(() => 0.5 - Math.random()); + const bucketNames = projectFixture.generateManyBucketNames(nBuckets).sort(() => 0.5 - Math.random()); return this.createMany(bucketNames); } @@ -80,12 +81,10 @@ class BucketUtility { } deleteMany(bucketNames) { - const promises = bucketNames.map(bucketName => - this.deleteOne(bucketName), - ); + const promises = bucketNames.map(bucketName => this.deleteOne(bucketName)); return Promise.all(promises); } - + /** * Recursively delete all versions of all objects within the bucket * @param bucketName @@ -97,33 +96,36 @@ class BucketUtility { let isTruncated = true; while (isTruncated) { - const response = await this.s3.send(new ListObjectVersionsCommand({ - Bucket: bucketName, - KeyMarker: keyMarker, - VersionIdMarker: versionIdMarker, - })); - - const objects = [ - ...(response.Versions || []), - ...(response.DeleteMarkers || []), - ].map(({ Key, VersionId }) => ({ Key, VersionId })); + const response = await this.s3.send( + new ListObjectVersionsCommand({ + Bucket: bucketName, + KeyMarker: keyMarker, + VersionIdMarker: versionIdMarker, + }), + ); + + const objects = [...(response.Versions || []), ...(response.DeleteMarkers || [])].map( + ({ Key, VersionId }) => ({ Key, VersionId }), + ); if (objects.length > 0) { try { - const result = await this.s3.send(new DeleteObjectsCommand({ - Bucket: bucketName, - Delete: { - Objects: objects, - Quiet: true - }, - ...(BypassGovernanceRetention && { BypassGovernanceRetention }), - })); + const result = await this.s3.send( + new DeleteObjectsCommand({ + Bucket: bucketName, + Delete: { + Objects: objects, + Quiet: true, + }, + ...(BypassGovernanceRetention && { BypassGovernanceRetention }), + }), + ); if (result.Errors && result.Errors.length > 0) { for (const e of result.Errors) { // eslint-disable-next-line no-console console.warn( `Warning BucketUtility.empty(): failed to delete s3://${bucketName}/${e.Key}` + - ` (versionId=${e.VersionId}): ${e.Code} - ${e.Message}` + ` (versionId=${e.VersionId}): ${e.Code} - ${e.Message}`, ); } } @@ -133,14 +135,18 @@ class BucketUtility { if (err.name !== 'BadDigest') { throw err; } - await Promise.all(objects.map(({ Key, VersionId }) => - this.s3.send(new DeleteObjectCommand({ - Bucket: bucketName, - Key, - VersionId, - ...(BypassGovernanceRetention && { BypassGovernanceRetention }), - })) - )); + await Promise.all( + objects.map(({ Key, VersionId }) => + this.s3.send( + new DeleteObjectCommand({ + Bucket: bucketName, + Key, + VersionId, + ...(BypassGovernanceRetention && { BypassGovernanceRetention }), + }), + ), + ), + ); } } @@ -153,12 +159,10 @@ class BucketUtility { } emptyMany(bucketNames) { - const promises = bucketNames.map( - bucketName => this.empty(bucketName) - ); + const promises = bucketNames.map(bucketName => this.empty(bucketName)); return Promise.all(promises); } - + emptyIfExists(bucketName) { return this.bucketExists(bucketName).then(exists => { if (exists) { @@ -169,14 +173,13 @@ class BucketUtility { } emptyManyIfExists(bucketNames) { - const promises = bucketNames.map(bucketName => - this.emptyIfExists(bucketName), - ); + const promises = bucketNames.map(bucketName => this.emptyIfExists(bucketName)); return Promise.all(promises); } getOwner() { - return this.s3.send(new ListBucketsCommand({})) + return this.s3 + .send(new ListBucketsCommand({})) .then(data => data.Owner) .catch(err => { throw err; diff --git a/tests/functional/aws-node-sdk/lib/utility/cors-util.js b/tests/functional/aws-node-sdk/lib/utility/cors-util.js index a1d52bf8cc..9e14d3fe6d 100644 --- a/tests/functional/aws-node-sdk/lib/utility/cors-util.js +++ b/tests/functional/aws-node-sdk/lib/utility/cors-util.js @@ -6,11 +6,9 @@ const conf = require('../../../../../lib/Config').config; const transport = conf.https ? https : http; const ipAddress = process.env.IP ? process.env.IP : '127.0.0.1'; -const hostname = process.env.AWS_ON_AIR ? 's3.amazonaws.com' : - ipAddress; +const hostname = process.env.AWS_ON_AIR ? 's3.amazonaws.com' : ipAddress; const port = process.env.AWS_ON_AIR ? 80 : 8000; - const statusCode = { 200: 200, 301: 301, // website redirect @@ -25,11 +23,10 @@ const statusCode = { }; function methodRequest(params, callback) { - const { method, bucket, objectKey, query, headers, code, - headersResponse, headersOmitted, isWebsite } = params; - const websiteHostname = process.env.S3_END_TO_END ? - `${bucket}.s3-website-us-east-1.scality.com` : - `${bucket}.s3-website-us-east-1.amazonaws.com`; + const { method, bucket, objectKey, query, headers, code, headersResponse, headersOmitted, isWebsite } = params; + const websiteHostname = process.env.S3_END_TO_END + ? `${bucket}.s3-website-us-east-1.scality.com` + : `${bucket}.s3-website-us-east-1.amazonaws.com`; const options = { port, @@ -59,32 +56,29 @@ function methodRequest(params, callback) { res.on('end', () => { const total = body.join(''); if (code) { - const message = Number.isNaN(parseInt(code, 10)) ? - `${code}` : ''; + const message = Number.isNaN(parseInt(code, 10)) ? `${code}` : ''; assert(total.indexOf(message) > -1, `Expected ${message}`); - assert.deepEqual(res.statusCode, statusCode[code], - `status code expected: ${statusCode[code]}`); + assert.deepEqual(res.statusCode, statusCode[code], `status code expected: ${statusCode[code]}`); } if (headersResponse) { Object.keys(headersResponse).forEach(key => { - assert.deepEqual(res.headers[key], headersResponse[key], - `error header: ${key}`); + assert.deepEqual(res.headers[key], headersResponse[key], `error header: ${key}`); }); } else { - // if no headersResponse provided, should not have these headers - // in the request - ['access-control-allow-origin', + // if no headersResponse provided, should not have these headers + // in the request + [ + 'access-control-allow-origin', 'access-control-allow-methods', 'access-control-allow-credentials', - 'vary'].forEach(key => { - assert.strictEqual(res.headers[key], undefined, - `Error: ${key} should not have value`); - }); + 'vary', + ].forEach(key => { + assert.strictEqual(res.headers[key], undefined, `Error: ${key} should not have value`); + }); } if (headersOmitted) { headersOmitted.forEach(key => { - assert.strictEqual(res.headers[key], undefined, - `Error: ${key} should not have value`); + assert.strictEqual(res.headers[key], undefined, `Error: ${key} should not have value`); }); } return callback(); diff --git a/tests/functional/aws-node-sdk/lib/utility/createEncryptedBucket.js b/tests/functional/aws-node-sdk/lib/utility/createEncryptedBucket.js index efb678f9c8..0b798d9140 100644 --- a/tests/functional/aws-node-sdk/lib/utility/createEncryptedBucket.js +++ b/tests/functional/aws-node-sdk/lib/utility/createEncryptedBucket.js @@ -13,27 +13,29 @@ function safeJSONParse(s) { } function createEncryptedBucket(bucketParams, cb) { - process.stdout.write('Creating encrypted bucket' + - `${bucketParams.Bucket}`); + process.stdout.write('Creating encrypted bucket' + `${bucketParams.Bucket}`); const config = getConfig(); const endpointWithoutHttp = config.endpoint.split('//')[1]; const host = endpointWithoutHttp.split(':')[0]; const port = endpointWithoutHttp.split(':')[1]; let locationConstraint; - if (bucketParams.CreateBucketConfiguration && - bucketParams.CreateBucketConfiguration.LocationConstraint) { - locationConstraint = bucketParams.CreateBucketConfiguration - .LocationConstraint; + if (bucketParams.CreateBucketConfiguration && bucketParams.CreateBucketConfiguration.LocationConstraint) { + locationConstraint = bucketParams.CreateBucketConfiguration.LocationConstraint; } const prog = `${__dirname}/../../../../../bin/create_encrypted_bucket.js`; let args = [ prog, - '-a', config.credentials.accessKeyId, - '-k', config.credentials.secretAccessKey, - '-b', bucketParams.Bucket, - '-h', host, - '-p', port, + '-a', + config.credentials.accessKeyId, + '-k', + config.credentials.secretAccessKey, + '-b', + bucketParams.Bucket, + '-h', + host, + '-p', + port, '-v', ]; if (locationConstraint) { @@ -43,24 +45,27 @@ function createEncryptedBucket(bucketParams, cb) { args = args.concat('-s'); } const body = []; - const child = childProcess.spawn(args[0], args) - .on('exit', () => { - const hasSucceed = body.join('').split('\n').find(item => { - const json = safeJSONParse(item); - const test = !(json instanceof Error) && json.name === 'S3' && - json.statusCode === 200; - if (test) { - return true; + const child = childProcess + .spawn(args[0], args) + .on('exit', () => { + const hasSucceed = body + .join('') + .split('\n') + .find(item => { + const json = safeJSONParse(item); + const test = !(json instanceof Error) && json.name === 'S3' && json.statusCode === 200; + if (test) { + return true; + } + return false; + }); + if (!hasSucceed) { + process.stderr.write(`${body.join('')}\n`); + return cb(new Error('Cannot create encrypted bucket')); } - return false; - }); - if (!hasSucceed) { - process.stderr.write(`${body.join('')}\n`); - return cb(new Error('Cannot create encrypted bucket')); - } - return cb(); - }) - .on('error', cb); + return cb(); + }) + .on('error', cb); child.stdout.on('data', chunk => body.push(chunk.toString())); } diff --git a/tests/functional/aws-node-sdk/lib/utility/customS3Request.js b/tests/functional/aws-node-sdk/lib/utility/customS3Request.js index d9c748cdd6..950106d303 100644 --- a/tests/functional/aws-node-sdk/lib/utility/customS3Request.js +++ b/tests/functional/aws-node-sdk/lib/utility/customS3Request.js @@ -6,7 +6,6 @@ const getConfig = require('../../test/support/config'); const config = getConfig('default'); const customRequestMiddleware = buildParams => next => async args => { - const { headers, query } = buildParams; const prevReq = args.request; @@ -35,10 +34,11 @@ const customRequestMiddleware = buildParams => next => async args => { async function customS3Request(CommandClass, params, buildParams) { const customS3 = new S3Client({ ...config }); - customS3.middlewareStack.add( - customRequestMiddleware(buildParams), - { step: 'build', name: 'customRequestMiddleware', tags: ['CUSTOM'] } - ); + customS3.middlewareStack.add(customRequestMiddleware(buildParams), { + step: 'build', + name: 'customRequestMiddleware', + tags: ['CUSTOM'], + }); const command = new CommandClass(params); const response = await customS3.send(command); @@ -50,7 +50,6 @@ async function customS3Request(CommandClass, params, buildParams) { }; return resData; - } module.exports = customS3Request; diff --git a/tests/functional/aws-node-sdk/lib/utility/genMaxSizeMetaHeaders.js b/tests/functional/aws-node-sdk/lib/utility/genMaxSizeMetaHeaders.js index d2ebeff178..65e402ac47 100644 --- a/tests/functional/aws-node-sdk/lib/utility/genMaxSizeMetaHeaders.js +++ b/tests/functional/aws-node-sdk/lib/utility/genMaxSizeMetaHeaders.js @@ -3,12 +3,10 @@ const constants = require('../../../../../constants'); function genMaxSizeMetaHeaders() { const metaHeaders = {}; const counter = 8; - const bytesPerHeader = - (constants.maximumMetaHeadersSize / counter); + const bytesPerHeader = constants.maximumMetaHeadersSize / counter; for (let i = 0; i < counter; i++) { const key = `header${i}`; - const valueLength = bytesPerHeader - - ('x-amz-meta-'.length + key.length); + const valueLength = bytesPerHeader - ('x-amz-meta-'.length + key.length); metaHeaders[key] = '0'.repeat(valueLength); } return metaHeaders; diff --git a/tests/functional/aws-node-sdk/lib/utility/provideRawOutput.js b/tests/functional/aws-node-sdk/lib/utility/provideRawOutput.js index 9e508f0c44..be9196832f 100644 --- a/tests/functional/aws-node-sdk/lib/utility/provideRawOutput.js +++ b/tests/functional/aws-node-sdk/lib/utility/provideRawOutput.js @@ -27,15 +27,13 @@ function provideRawOutput(args, cb) { httpCode = lines.find(line => { const trimmed = line.trim().toUpperCase(); // ignore 100 Continue HTTP code - if (trimmed.startsWith('HTTP/1.1 ') && - !trimmed.includes('100 CONTINUE')) { + if (trimmed.startsWith('HTTP/1.1 ') && !trimmed.includes('100 CONTINUE')) { return true; } return false; }); if (httpCode) { - httpCode = httpCode.trim().replace('HTTP/1.1 ', '') - .toUpperCase(); + httpCode = httpCode.trim().replace('HTTP/1.1 ', '').toUpperCase(); } } return cb(httpCode, procData); @@ -46,8 +44,6 @@ function provideRawOutput(args, cb) { } provideRawOutput[util.promisify.custom] = args => - new Promise(resolve => - provideRawOutput(args, (httpCode, rawOutput) => resolve({ httpCode, rawOutput })) - ); + new Promise(resolve => provideRawOutput(args, (httpCode, rawOutput) => resolve({ httpCode, rawOutput }))); module.exports = provideRawOutput; diff --git a/tests/functional/aws-node-sdk/lib/utility/replication.js b/tests/functional/aws-node-sdk/lib/utility/replication.js index 7f1f466af9..f50cefbe0c 100644 --- a/tests/functional/aws-node-sdk/lib/utility/replication.js +++ b/tests/functional/aws-node-sdk/lib/utility/replication.js @@ -1,16 +1,6 @@ const replicationUtils = { - requiredConfigProperties: [ - 'Role', - 'Rules', - 'Status', - 'Destination', - 'Bucket', - ], - optionalConfigProperties: [ - 'ID', - 'StorageClass', - 'Prefix', - ], + requiredConfigProperties: ['Role', 'Rules', 'Status', 'Destination', 'Bucket'], + optionalConfigProperties: ['ID', 'StorageClass', 'Prefix'], invalidRoleARNs: [ '', '*:aws:iam::account-id:role/resource', @@ -33,7 +23,6 @@ const replicationUtils = { 'arn:aws:iam::ac:role', 'arn:aws:iam::a c:role', 'arn:aws:iam::*:role', - ], invalidBucketARNs: [ '', @@ -46,18 +35,9 @@ const replicationUtils = { 'arn:aws:s3:::*', 'arn:aws:s3:::invalidBucketName', ], - validStatuses: [ - 'Enabled', - 'Disabled', - ], - validStorageClasses: [ - 'STANDARD', - 'STANDARD_IA', - 'REDUCED_REDUNDANCY', - ], - validMultipleStorageClasses: [ - 'zenko,us-east-2', - ], + validStatuses: ['Enabled', 'Disabled'], + validStorageClasses: ['STANDARD', 'STANDARD_IA', 'REDUCED_REDUNDANCY'], + validMultipleStorageClasses: ['zenko,us-east-2'], }; module.exports = replicationUtils; diff --git a/tests/functional/aws-node-sdk/lib/utility/tagging.js b/tests/functional/aws-node-sdk/lib/utility/tagging.js index 9039a9b5eb..f215fcb9fc 100644 --- a/tests/functional/aws-node-sdk/lib/utility/tagging.js +++ b/tests/functional/aws-node-sdk/lib/utility/tagging.js @@ -1,13 +1,14 @@ const taggingTests = [ - { tag: { key: '+- =._:/', value: '+- =._:/' }, - it: 'should return tags if tags are valid' }, - { tag: { key: 'key1', value: '' }, - it: 'should return tags if value is an empty string' }, - { tag: { key: 'w'.repeat(129), value: 'foo' }, + { tag: { key: '+- =._:/', value: '+- =._:/' }, it: 'should return tags if tags are valid' }, + { tag: { key: 'key1', value: '' }, it: 'should return tags if value is an empty string' }, + { + tag: { key: 'w'.repeat(129), value: 'foo' }, error: 'InvalidTag', code: 400, - it: 'should return InvalidTag if key length is greater than 128' }, - { tag: { key: 'bar', value: 'f'.repeat(257) }, + it: 'should return InvalidTag if key length is greater than 128', + }, + { + tag: { key: 'bar', value: 'f'.repeat(257) }, error: 'InvalidTag', code: 400, it: 'should return InvalidTag if key length is greater than 256', diff --git a/tests/functional/aws-node-sdk/lib/utility/test-utils.js b/tests/functional/aws-node-sdk/lib/utility/test-utils.js index 8e90ab8e95..f4e8c7cdc4 100644 --- a/tests/functional/aws-node-sdk/lib/utility/test-utils.js +++ b/tests/functional/aws-node-sdk/lib/utility/test-utils.js @@ -15,7 +15,7 @@ function hasLocation(lc) { * In our implementation, it's the only way to move objects to cold storage (can't write directly to cold) * Even if Transition could be used without cold storage location (hot transition), but both features are * enabled together at the moment. -*/ + */ const hasColdStorage = config.supportedLifecycleRules.some(rule => rule.endsWith('Transition')); module.exports = { diff --git a/tests/functional/aws-node-sdk/lib/utility/website-util.js b/tests/functional/aws-node-sdk/lib/utility/website-util.js index 463403b525..605112823d 100644 --- a/tests/functional/aws-node-sdk/lib/utility/website-util.js +++ b/tests/functional/aws-node-sdk/lib/utility/website-util.js @@ -3,16 +3,17 @@ const async = require('async'); const fs = require('fs'); const path = require('path'); const url = require('url'); -const { CreateBucketCommand, - DeleteBucketCommand, +const { + CreateBucketCommand, + DeleteBucketCommand, PutBucketWebsiteCommand, - DeleteObjectCommand, - PutObjectCommand } = require('@aws-sdk/client-s3'); + DeleteObjectCommand, + PutObjectCommand, +} = require('@aws-sdk/client-s3'); const { makeRequest } = require('../../../raw-node/utils/makeRequest'); -const bucketName = process.env.AWS_ON_AIR ? 'awsbucketwebsitetester' : - 'bucketwebsitetester'; +const bucketName = process.env.AWS_ON_AIR ? 'awsbucketwebsitetester' : 'bucketwebsitetester'; let awsCredentials; function _parseConfigValue(string, fileSlice) { @@ -36,10 +37,8 @@ function _retrieveAWSCredentials(profile) { const fileContents = file.split('\n'); const profileIndex = file.indexOf(`[${profile}]`); if (profileIndex > -1) { - const accessKey = _parseConfigValue('aws_access_key_id', - fileContents.slice(profileIndex)); - const secretKey = _parseConfigValue('aws_secret_access_key', - fileContents.slice(profileIndex)); + const accessKey = _parseConfigValue('aws_access_key_id', fileContents.slice(profileIndex)); + const secretKey = _parseConfigValue('aws_secret_access_key', fileContents.slice(profileIndex)); return { accessKey, secretKey }; } const msg = `Profile ${profile} does not exist in AWS credential file`; @@ -75,46 +74,41 @@ function _assertResponseHtml(response, elemtag, content) { const startIndex = response.indexOf(startingTag); const endIndex = response.indexOf(''); assert(startIndex > -1 && endIndex > -1, 'Did not find ul element'); - const ulElem = response.slice(startIndex + startingTag.length, - endIndex); + const ulElem = response.slice(startIndex + startingTag.length, endIndex); content.forEach(item => { _assertResponseHtml(ulElem, 'li', item); }); } else { const elem = `<${elemtag}>${content}`; - assert(response.includes(elem), - `Expected but did not find '${elem}' in html`); + assert(response.includes(elem), `Expected but did not find '${elem}' in html`); } } function _assertContainsHtml(responseBody) { - assert(responseBody.startsWith('') && - responseBody.includes(''), 'Did not find html tags'); + assert(responseBody.startsWith('') && responseBody.includes(''), 'Did not find html tags'); } function _assertResponseHtml404(method, response, type) { assert.strictEqual(response.statusCode, 404); if (method === 'HEAD') { if (type === '404-no-such-bucket') { - assert.strictEqual(response.headers['x-amz-error-code'], - 'NoSuchBucket'); + assert.strictEqual(response.headers['x-amz-error-code'], 'NoSuchBucket'); // Need arsenal fixed to remove period at the end // so compatible with aws - assert.strictEqual(response.headers['x-amz-error-message'], - 'The specified bucket does not exist.'); + assert.strictEqual(response.headers['x-amz-error-message'], 'The specified bucket does not exist.'); } else if (type === '404-no-such-website-configuration') { - assert.strictEqual(response.headers['x-amz-error-code'], - 'NoSuchWebsiteConfiguration'); - assert.strictEqual(response.headers['x-amz-error-message'], - 'The specified bucket does not have a website configuration'); + assert.strictEqual(response.headers['x-amz-error-code'], 'NoSuchWebsiteConfiguration'); + assert.strictEqual( + response.headers['x-amz-error-message'], + 'The specified bucket does not have a website configuration', + ); } else if (type === '404-not-found') { - assert.strictEqual(response.headers['x-amz-error-code'], - 'NoSuchKey'); - assert.strictEqual(response.headers['x-amz-error-message'], - 'The specified key does not exist.'); + assert.strictEqual(response.headers['x-amz-error-code'], 'NoSuchKey'); + assert.strictEqual(response.headers['x-amz-error-message'], 'The specified key does not exist.'); } else { - throw new Error(`'${type}' is not a recognized 404 ` + - 'error checked in the WebsiteConfigTester.checkHTML function'); + throw new Error( + `'${type}' is not a recognized 404 ` + 'error checked in the WebsiteConfigTester.checkHTML function', + ); } // don't need to check HTML for head requests return; @@ -131,18 +125,15 @@ function _assertResponseHtml404(method, response, type) { } else if (type === '404-no-such-website-configuration') { _assertResponseHtml(response.body, 'ul', [ 'Code: NoSuchWebsiteConfiguration', - 'Message: The specified bucket does not have a ' + - 'website configuration', + 'Message: The specified bucket does not have a ' + 'website configuration', `BucketName: ${bucketName}`, ]); } else if (type === '404-not-found') { - _assertResponseHtml(response.body, 'ul', [ - 'Code: NoSuchKey', - 'Message: The specified key does not exist.', - ]); + _assertResponseHtml(response.body, 'ul', ['Code: NoSuchKey', 'Message: The specified key does not exist.']); } else { - throw new Error(`'${type}' is not a recognized 404 ` + - 'error checked in the WebsiteConfigTester.checkHTML function'); + throw new Error( + `'${type}' is not a recognized 404 ` + 'error checked in the WebsiteConfigTester.checkHTML function', + ); } } @@ -150,38 +141,35 @@ function _assertResponseHtml403(method, response, type) { assert.strictEqual(response.statusCode, 403); if (method === 'HEAD') { if (type === '403-access-denied') { - assert.strictEqual(response.headers['x-amz-error-code'], - 'AccessDenied'); - assert.strictEqual(response.headers['x-amz-error-message'], - 'Access Denied'); + assert.strictEqual(response.headers['x-amz-error-code'], 'AccessDenied'); + assert.strictEqual(response.headers['x-amz-error-message'], 'Access Denied'); } else if (type !== '403-retrieve-error-document') { - throw new Error(`'${type}' is not a recognized 403 ` + - 'error checked in the WebsiteConfigTester.checkHTML function'); + throw new Error( + `'${type}' is not a recognized 403 ` + 'error checked in the WebsiteConfigTester.checkHTML function', + ); } } else { _assertContainsHtml(response.body); _assertResponseHtml(response.body, 'title', '403 Forbidden'); _assertResponseHtml(response.body, 'h1', '403 Forbidden'); - _assertResponseHtml(response.body, 'ul', [ - 'Code: AccessDenied', - 'Message: Access Denied', - ]); + _assertResponseHtml(response.body, 'ul', ['Code: AccessDenied', 'Message: Access Denied']); if (type === '403-retrieve-error-document') { - _assertResponseHtml(response.body, 'h3', - 'An Error Occurred While Attempting to ' + - 'Retrieve a Custom Error Document'); + _assertResponseHtml( + response.body, + 'h3', + 'An Error Occurred While Attempting to ' + 'Retrieve a Custom Error Document', + ); // start searching for second `ul` element after `h3` element const startingTag = ''; - const startIndex = response.body.indexOf(startingTag) - + startingTag.length; - _assertResponseHtml(response.body.slice(startIndex), - 'ul', [ + const startIndex = response.body.indexOf(startingTag) + startingTag.length; + _assertResponseHtml(response.body.slice(startIndex), 'ul', [ 'Code: AccessDenied', 'Message: Access Denied', ]); } else if (type !== '403-access-denied') { - throw new Error(`'${type}' is not a recognized 403 ` + - 'error checked in the WebsiteConfigTester.checkHTML function'); + throw new Error( + `'${type}' is not a recognized 403 ` + 'error checked in the WebsiteConfigTester.checkHTML function', + ); } } } @@ -192,22 +180,17 @@ function _assertResponseHtmlErrorUser(response, type) { } else if (type === 'error-user-404') { assert.strictEqual(response.statusCode, 404); } - _assertResponseHtml(response.body, 'title', - 'Error!!'); - _assertResponseHtml(response.body, 'h1', - 'It appears you messed up'); + _assertResponseHtml(response.body, 'title', 'Error!!'); + _assertResponseHtml(response.body, 'h1', 'It appears you messed up'); } function _assertResponseHtmlIndexUser(response) { assert.strictEqual(response.statusCode, 200); - _assertResponseHtml(response.body, 'title', - 'Best testing website ever'); - _assertResponseHtml(response.body, 'h1', 'Welcome to my ' + - 'extraordinary bucket website testing page'); + _assertResponseHtml(response.body, 'title', 'Best testing website ever'); + _assertResponseHtml(response.body, 'h1', 'Welcome to my ' + 'extraordinary bucket website testing page'); } -function _assertResponseHtmlRedirect(response, type, redirectUrl, method, - expectedHeaders) { +function _assertResponseHtmlRedirect(response, type, redirectUrl, method, expectedHeaders) { if (type === 'redirect' || type === 'redirect-user') { assert.strictEqual(response.statusCode, 301); assert.strictEqual(response.body, ''); @@ -218,13 +201,10 @@ function _assertResponseHtmlRedirect(response, type, redirectUrl, method, return; // no need to check HTML } - _assertResponseHtml(response.body, 'title', - 'Best redirect link ever'); - _assertResponseHtml(response.body, 'h1', - 'Welcome to your redirection file'); + _assertResponseHtml(response.body, 'title', 'Best redirect link ever'); + _assertResponseHtml(response.body, 'h1', 'Welcome to your redirection file'); } else if (type.startsWith('redirect-error')) { - assert.strictEqual(response.statusCode, - type === 'redirect-error-found' ? 302 : 301); + assert.strictEqual(response.statusCode, type === 'redirect-error-found' ? 302 : 301); assert.strictEqual(response.headers.location, redirectUrl); Object.entries(expectedHeaders || {}).forEach(([key, val]) => { assert.strictEqual(response.headers[key], val); @@ -232,21 +212,18 @@ function _assertResponseHtmlRedirect(response, type, redirectUrl, method, if (type === 'redirect-error-found') { assert.strictEqual(response.headers['x-amz-error-code'], 'Found'); - assert.strictEqual(response.headers['x-amz-error-message'], - 'Resource Found'); + assert.strictEqual(response.headers['x-amz-error-message'], 'Resource Found'); _assertContainsHtml(response.body); _assertResponseHtml(response.body, 'title', '302 Found'); _assertResponseHtml(response.body, 'h1', '302 Found'); - _assertResponseHtml(response.body, 'ul', [ - 'Code: Found', - 'Message: Resource Found', - ]); + _assertResponseHtml(response.body, 'ul', ['Code: Found', 'Message: Resource Found']); } else { _assertResponseHtmlErrorUser(response, type); } } else { - throw new Error(`'${type}' is not a recognized redirect type ` + - 'checked in the WebsiteConfigTester.checkHTML function'); + throw new Error( + `'${type}' is not a recognized redirect type ` + 'checked in the WebsiteConfigTester.checkHTML function', + ); } } @@ -285,21 +262,20 @@ class WebsiteConfigTester { } /** checkHTML - check response for website head or get - * @param {object} params - function params - * @param {string} params.method - type of website request, 'HEAD' or 'GET' - * @param {string} params.responseType - type of response expected - * @param {string} [params.auth] - whether to use valid or invalid auth - * crendentials: 'valid credentials' or 'invalid credentials' - * @param {string} [params.url] - request url - * @param {string} [params.redirectUrl] - redirect - * @param {object} [params.expectedHeaders] - expected headers in response - * with expected values (e.g., {x-amz-error-code: AccessDenied}) - * @param {function} callback - callback - * @return {undefined} - */ + * @param {object} params - function params + * @param {string} params.method - type of website request, 'HEAD' or 'GET' + * @param {string} params.responseType - type of response expected + * @param {string} [params.auth] - whether to use valid or invalid auth + * crendentials: 'valid credentials' or 'invalid credentials' + * @param {string} [params.url] - request url + * @param {string} [params.redirectUrl] - redirect + * @param {object} [params.expectedHeaders] - expected headers in response + * with expected values (e.g., {x-amz-error-code: AccessDenied}) + * @param {function} callback - callback + * @return {undefined} + */ static checkHTML(params, callback) { - const { method, responseType, auth, url, redirectUrl, expectedHeaders } - = params; + const { method, responseType, auth, url, redirectUrl, expectedHeaders } = params; _makeWebsiteRequest(auth, method, url, (err, res) => { assert.strictEqual(err, null, `Unexpected request err ${err}`); if (responseType) { @@ -310,20 +286,20 @@ class WebsiteConfigTester { } else if (responseType.startsWith('error-user')) { _assertResponseHtmlErrorUser(res, responseType); } else if (responseType.startsWith('redirect')) { - _assertResponseHtmlRedirect(res, responseType, redirectUrl, - method, expectedHeaders); + _assertResponseHtmlRedirect(res, responseType, redirectUrl, method, expectedHeaders); if (responseType === 'redirect-user') { process.stdout.write('Following redirect location\n'); - return this.checkHTML({ method, - url: res.headers.location, - responseType: 'redirected-user' }, - callback); + return this.checkHTML( + { method, url: res.headers.location, responseType: 'redirected-user' }, + callback, + ); } } else if (responseType === 'index-user') { _assertResponseHtmlIndexUser(res); } else { - throw new Error(`'${responseType}' is not a response ` + - 'type recognized by WebsiteConfigTester.checkHTML'); + throw new Error( + `'${responseType}' is not a response ` + 'type recognized by WebsiteConfigTester.checkHTML', + ); } } return callback(); @@ -341,48 +317,67 @@ class WebsiteConfigTester { * @param {function} cb - callback to end test * @return {undefined} */ - static makeHeadRequest(auth, url, expectedStatusCode, expectedHeaders, - cb) { + static makeHeadRequest(auth, url, expectedStatusCode, expectedHeaders, cb) { _makeWebsiteRequest(auth, 'HEAD', url, (err, res) => { // body should be empty assert.deepStrictEqual(res.body, ''); assert.strictEqual(res.statusCode, expectedStatusCode); const headers = Object.keys(expectedHeaders); headers.forEach(header => { - assert.strictEqual(res.headers[header], - expectedHeaders[header]); + assert.strictEqual(res.headers[header], expectedHeaders[header]); }); return cb(); }); } static createPutBucketWebsite(s3, bucket, bucketACL, objects, done) { - s3.send(new CreateBucketCommand({ Bucket: bucket, ACL: bucketACL })).then(() => { - const webConfig = new WebsiteConfigTester('index.html', - 'error.html'); - return s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })).then(() => async.forEachOf(objects, - (acl, object, next) => { - s3.send(new PutObjectCommand({ Bucket: bucket, - Key: `${object}.html`, - ACL: acl, - Body: fs.readFileSync(path.join(__dirname, - `/../../test/object/websiteFiles/${object}.html`)), - })).then(() => next()).catch(next); - }, done)); - }).catch(err => done(err)); + s3.send(new CreateBucketCommand({ Bucket: bucket, ACL: bucketACL })) + .then(() => { + const webConfig = new WebsiteConfigTester('index.html', 'error.html'); + return s3 + .send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })) + .then(() => + async.forEachOf( + objects, + (acl, object, next) => { + s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: `${object}.html`, + ACL: acl, + Body: fs.readFileSync( + path.join(__dirname, `/../../test/object/websiteFiles/${object}.html`), + ), + }), + ) + .then(() => next()) + .catch(next); + }, + done, + ), + ); + }) + .catch(err => done(err)); } static deleteObjectsThenBucket(s3, bucket, objects, done) { - async.forEachOf(objects, (acl, object, next) => { - s3.send(new DeleteObjectCommand({ Bucket: bucket, - Key: `${object}.html` })).then(() => next()).catch(next); - }, err => { - if (err) { - return done(err); - } - return s3.send(new DeleteBucketCommand({ Bucket: bucket })).then(() => done()).catch(done); - }); + async.forEachOf( + objects, + (acl, object, next) => { + s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: `${object}.html` })) + .then(() => next()) + .catch(next); + }, + err => { + if (err) { + return done(err); + } + return s3 + .send(new DeleteBucketCommand({ Bucket: bucket })) + .then(() => done()) + .catch(done); + }, + ); } } diff --git a/tests/functional/aws-node-sdk/schema/bucket.json b/tests/functional/aws-node-sdk/schema/bucket.json index ee69950078..1d6b3933cd 100644 --- a/tests/functional/aws-node-sdk/schema/bucket.json +++ b/tests/functional/aws-node-sdk/schema/bucket.json @@ -1,94 +1,79 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", - "id": "http://jsonschema.net", - "type": "object", - "properties": { - "Contents": { - "id": "http://jsonschema.net/Contents", - "type": "array", - "minItems": 0, - "items": { - "id": "http://jsonschema.net/Buckets/0", - "type": "object", - "properties": { - "Key": { - "id": "http://jsonschema.net/Contents/0/Key", + "$schema": "http://json-schema.org/draft-04/schema#", + "id": "http://jsonschema.net", + "type": "object", + "properties": { + "Contents": { + "id": "http://jsonschema.net/Contents", + "type": "array", + "minItems": 0, + "items": { + "id": "http://jsonschema.net/Buckets/0", + "type": "object", + "properties": { + "Key": { + "id": "http://jsonschema.net/Contents/0/Key", + "type": "string" + }, + "LastModified": { + "id": "http://jsonschema.net/Contents/0/LastModified", + "type": "object" + }, + "ETag": { + "id": "http://jsonschema.net/Contents/0/ETag", + "type": "string" + }, + "Size": { + "id": "http://jsonschema.net/Contents/0/Size", + "type": "integer" + }, + "StorageClass": { + "id": "http://jsonschema.net/Contents/0/StorageClass", + "enum": ["STANDARD", "REDUCED_REDUNDANCY", "GLACIER"] + }, + "Owner": { + "id": "http://jsonschema.net/Contents/0/Owner", + "type": "object", + "properties": { + "DisplayName": { + "id": "http://jsonschema.net/Contents/0/Owner/DisplayName", + "type": "string" + }, + "ID": { + "id": "http://jsonschema.net/Contents/0/Owner/ID", + "type": "string" + } + }, + "required": ["DisplayName", "ID"] + } + }, + "required": ["Key", "LastModified", "ETag", "Size", "StorageClass", "Owner"] + } + }, + "Marker": { + "id": "http://jsonschema.net/Marker", + "type": "string" + }, + "Name": { + "id": "http://jsonschema.net/Name", "type": "string" - }, - "LastModified": { - "id": "http://jsonschema.net/Contents/0/LastModified", - "type": "object" - }, - "ETag": { - "id": "http://jsonschema.net/Contents/0/ETag", + }, + "Prefix": { + "id": "http://jsonschema.net/Prefix", "type": "string" - }, - "Size": { - "id": "http://jsonschema.net/Contents/0/Size", + }, + "MaxKeys": { + "id": "http://jsonschema.net/MaxKeys", "type": "integer" - }, - "StorageClass": { - "id": "http://jsonschema.net/Contents/0/StorageClass", - "enum": [ "STANDARD", "REDUCED_REDUNDANCY", "GLACIER" ] - }, - "Owner": { - "id": "http://jsonschema.net/Contents/0/Owner", - "type": "object", - "properties": { - "DisplayName": { - "id": "http://jsonschema.net/Contents/0/Owner/DisplayName", - "type": "string" - }, - "ID": { - "id": "http://jsonschema.net/Contents/0/Owner/ID", - "type": "string" - } - }, - "required": [ "DisplayName", "ID" ] - } }, - "required": [ - "Key", - "LastModified", - "ETag", - "Size", - "StorageClass", - "Owner" - ] - } - }, - "Marker": { - "id": "http://jsonschema.net/Marker", - "type": "string" - }, - "Name": { - "id": "http://jsonschema.net/Name", - "type": "string" - }, - "Prefix": { - "id": "http://jsonschema.net/Prefix", - "type": "string" - }, - "MaxKeys": { - "id": "http://jsonschema.net/MaxKeys", - "type": "integer" - }, - "CommonPrefixes": { - "id": "http://jsonschema.net/CommonPrefixes", - "type": "array" + "CommonPrefixes": { + "id": "http://jsonschema.net/CommonPrefixes", + "type": "array" + }, + "IsTruncated": { + "id": "http://jsonschema.net/IsTruncated", + "type": "boolean" + } }, - "IsTruncated": { - "id": "http://jsonschema.net/IsTruncated", - "type": "boolean" - } - }, - "required": [ - "IsTruncated", - "Marker", - "Contents", - "Name", - "Prefix", - "MaxKeys", - "CommonPrefixes" - ] + "required": ["IsTruncated", "Marker", "Contents", "Name", "Prefix", "MaxKeys", "CommonPrefixes"] } diff --git a/tests/functional/aws-node-sdk/schema/bucketV2.json b/tests/functional/aws-node-sdk/schema/bucketV2.json index c868be3318..0594061d1c 100644 --- a/tests/functional/aws-node-sdk/schema/bucketV2.json +++ b/tests/functional/aws-node-sdk/schema/bucketV2.json @@ -1,104 +1,91 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", - "id": "http://jsonschema.net", - "type": "object", - "properties": { - "Contents": { - "id": "http://jsonschema.net/Contents", - "type": "array", - "minItems": 0, - "items": { - "id": "http://jsonschema.net/Buckets/0", - "type": "object", - "properties": { - "Key": { - "id": "http://jsonschema.net/Contents/0/Key", + "$schema": "http://json-schema.org/draft-04/schema#", + "id": "http://jsonschema.net", + "type": "object", + "properties": { + "Contents": { + "id": "http://jsonschema.net/Contents", + "type": "array", + "minItems": 0, + "items": { + "id": "http://jsonschema.net/Buckets/0", + "type": "object", + "properties": { + "Key": { + "id": "http://jsonschema.net/Contents/0/Key", + "type": "string" + }, + "LastModified": { + "id": "http://jsonschema.net/Contents/0/LastModified", + "type": "object" + }, + "ETag": { + "id": "http://jsonschema.net/Contents/0/ETag", + "type": "string" + }, + "Size": { + "id": "http://jsonschema.net/Contents/0/Size", + "type": "integer" + }, + "StorageClass": { + "id": "http://jsonschema.net/Contents/0/StorageClass", + "enum": ["STANDARD", "REDUCED_REDUNDANCY", "GLACIER"] + }, + "Owner": { + "id": "http://jsonschema.net/Contents/0/Owner", + "type": "object", + "properties": { + "DisplayName": { + "id": "http://jsonschema.net/Contents/0/Owner/DisplayName", + "type": "string" + }, + "ID": { + "id": "http://jsonschema.net/Contents/0/Owner/ID", + "type": "string" + } + }, + "required": ["DisplayName", "ID"] + } + }, + "required": ["Key", "LastModified", "ETag", "Size", "StorageClass"] + } + }, + "StartAfter": { + "id": "http://jsonschema.net/StartAfter", + "type": "string" + }, + "ContinuationToken": { + "id": "http://jsonschema.net/ContinuationToken", + "type": "string" + }, + "NextContinuationToken": { + "id": "http://jsonschema.net/NextContinuationToken", "type": "string" - }, - "LastModified": { - "id": "http://jsonschema.net/Contents/0/LastModified", - "type": "object" - }, - "ETag": { - "id": "http://jsonschema.net/Contents/0/ETag", + }, + "Name": { + "id": "http://jsonschema.net/Name", + "type": "string" + }, + "Prefix": { + "id": "http://jsonschema.net/Prefix", "type": "string" - }, - "Size": { - "id": "http://jsonschema.net/Contents/0/Size", + }, + "KeyCount": { + "id": "http://jsonschema.net/KeyCount", "type": "integer" - }, - "StorageClass": { - "id": "http://jsonschema.net/Contents/0/StorageClass", - "enum": [ "STANDARD", "REDUCED_REDUNDANCY", "GLACIER" ] - }, - "Owner": { - "id": "http://jsonschema.net/Contents/0/Owner", - "type": "object", - "properties": { - "DisplayName": { - "id": "http://jsonschema.net/Contents/0/Owner/DisplayName", - "type": "string" - }, - "ID": { - "id": "http://jsonschema.net/Contents/0/Owner/ID", - "type": "string" - } - }, - "required": [ "DisplayName", "ID" ] - } }, - "required": [ - "Key", - "LastModified", - "ETag", - "Size", - "StorageClass" - ] - } - }, - "StartAfter": { - "id": "http://jsonschema.net/StartAfter", - "type": "string" - }, - "ContinuationToken": { - "id": "http://jsonschema.net/ContinuationToken", - "type": "string" - }, - "NextContinuationToken": { - "id": "http://jsonschema.net/NextContinuationToken", - "type": "string" - }, - "Name": { - "id": "http://jsonschema.net/Name", - "type": "string" - }, - "Prefix": { - "id": "http://jsonschema.net/Prefix", - "type": "string" - }, - "KeyCount": { - "id": "http://jsonschema.net/KeyCount", - "type": "integer" - }, - "MaxKeys": { - "id": "http://jsonschema.net/MaxKeys", - "type": "integer" - }, - "CommonPrefixes": { - "id": "http://jsonschema.net/CommonPrefixes", - "type": "array" + "MaxKeys": { + "id": "http://jsonschema.net/MaxKeys", + "type": "integer" + }, + "CommonPrefixes": { + "id": "http://jsonschema.net/CommonPrefixes", + "type": "array" + }, + "IsTruncated": { + "id": "http://jsonschema.net/IsTruncated", + "type": "boolean" + } }, - "IsTruncated": { - "id": "http://jsonschema.net/IsTruncated", - "type": "boolean" - } - }, - "required": [ - "IsTruncated", - "Contents", - "Name", - "Prefix", - "MaxKeys", - "CommonPrefixes" - ] + "required": ["IsTruncated", "Contents", "Name", "Prefix", "MaxKeys", "CommonPrefixes"] } diff --git a/tests/functional/aws-node-sdk/schema/service.json b/tests/functional/aws-node-sdk/schema/service.json index 03b3c1daa6..8cb0a54b1c 100644 --- a/tests/functional/aws-node-sdk/schema/service.json +++ b/tests/functional/aws-node-sdk/schema/service.json @@ -1,48 +1,42 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", - "id": "http://jsonschema.net", - "type": "object", - "properties": { - "Buckets": { - "id": "http://jsonschema.net/Buckets", - "type": "array", - "minItems": 0, - "items": { - "id": "http://jsonschema.net/Buckets/0", - "type": "object", - "properties": { - "Name": { - "id": "http://jsonschema.net/Buckets/0/Name", - "type": "string" - }, - "CreationDate": { - "id": "http://jsonschema.net/Buckets/0/CreationDate", - "type": "object" - } + "$schema": "http://json-schema.org/draft-04/schema#", + "id": "http://jsonschema.net", + "type": "object", + "properties": { + "Buckets": { + "id": "http://jsonschema.net/Buckets", + "type": "array", + "minItems": 0, + "items": { + "id": "http://jsonschema.net/Buckets/0", + "type": "object", + "properties": { + "Name": { + "id": "http://jsonschema.net/Buckets/0/Name", + "type": "string" + }, + "CreationDate": { + "id": "http://jsonschema.net/Buckets/0/CreationDate", + "type": "object" + } + }, + "required": ["Name", "CreationDate"] + } }, - "required": [ - "Name", - "CreationDate" - ] - } - }, - "Owner": { - "id": "http://jsonschema.net/Owner", - "type": "object", - "properties": { - "DisplayName": { - "id": "http://jsonschema.net/Owner/DisplayName", - "type": "string" - }, - "ID": { - "id": "http://jsonschema.net/Owner/ID", - "type": "string" + "Owner": { + "id": "http://jsonschema.net/Owner", + "type": "object", + "properties": { + "DisplayName": { + "id": "http://jsonschema.net/Owner/DisplayName", + "type": "string" + }, + "ID": { + "id": "http://jsonschema.net/Owner/ID", + "type": "string" + } + } } - } - } - }, - "required": [ - "Buckets", - "Owner" - ] + }, + "required": ["Buckets", "Owner"] } diff --git a/tests/functional/aws-node-sdk/test/bucket/aclUsingPredefinedGroups.js b/tests/functional/aws-node-sdk/test/bucket/aclUsingPredefinedGroups.js index 88d0f72027..42b445bc09 100644 --- a/tests/functional/aws-node-sdk/test/bucket/aclUsingPredefinedGroups.js +++ b/tests/functional/aws-node-sdk/test/bucket/aclUsingPredefinedGroups.js @@ -31,7 +31,7 @@ withV4(sigCfg => { return otherAccountBucketUtil.s3.send(new Operation(params)); } else { const command = new Operation(params); - + // Create unsigned client const unsignedClient = new BucketUtility('default', { ...sigCfg, @@ -68,9 +68,9 @@ withV4(sigCfg => { step: 'serialize', priority: 'high', before: 'awsAuthMiddleware', - } + }, ); - } + }, }); return unsignedClient.s3.send(command); } @@ -102,8 +102,7 @@ withV4(sigCfg => { const grantUri = `uri=${auth ? constants.allAuthedUsersId : constants.publicId}`; // TODO fix flakiness on E2E and re-enable, see CLDSRV-254 - describeSkipIfE2E('PUT Bucket ACL using predefined groups - ' + - `${authType} request`, () => { + describeSkipIfE2E('PUT Bucket ACL using predefined groups - ' + `${authType} request`, () => { const aclParam = { Bucket: testBucket, ACL: 'private', @@ -111,11 +110,13 @@ withV4(sigCfg => { beforeEach(async () => { await s3.send(new CreateBucketCommand({ Bucket: testBucket })); - await s3.send(new PutObjectCommand({ - Bucket: testBucket, - Body: testBody, - Key: ownerObjKey, - })); + await s3.send( + new PutObjectCommand({ + Bucket: testBucket, + Body: testBody, + Key: ownerObjKey, + }), + ); }); afterEach(async () => { @@ -123,16 +124,24 @@ withV4(sigCfg => { await ownerAccountBucketUtil.deleteOne(testBucket); }); - it('should grant read access', () => s3.send(new PutBucketAclCommand({ - Bucket: testBucket, - GrantRead: grantUri, - })) + it('should grant read access', () => + s3 + .send( + new PutBucketAclCommand({ + Bucket: testBucket, + GrantRead: grantUri, + }), + ) .then(() => awsRequest(auth, ListObjectsV2Command, { Bucket: testBucket }))); - it('should grant read access with grant-full-control', () => s3.send(new PutBucketAclCommand({ - Bucket: testBucket, - GrantFullControl: grantUri, - })) + it('should grant read access with grant-full-control', () => + s3 + .send( + new PutBucketAclCommand({ + Bucket: testBucket, + GrantFullControl: grantUri, + }), + ) .then(() => awsRequest(auth, ListObjectsV2Command, { Bucket: testBucket }))); it('should not grant read access', done => { @@ -143,57 +152,81 @@ withV4(sigCfg => { // Don't return the promise! }); - it('should grant write access', () => s3.send(new PutBucketAclCommand({ - Bucket: testBucket, - GrantWrite: grantUri, - })) - .then(() => awsRequest(auth, PutObjectCommand, { - Bucket: testBucket, - Body: testBody, - Key: testKey, - }))); - - it('should grant write access with grant-full-control', () => s3.send(new PutBucketAclCommand({ - Bucket: testBucket, - GrantFullControl: grantUri, - })) - .then(() => awsRequest(auth, PutObjectCommand, { - Bucket: testBucket, - Body: testBody, - Key: testKey, - }))); + it('should grant write access', () => + s3 + .send( + new PutBucketAclCommand({ + Bucket: testBucket, + GrantWrite: grantUri, + }), + ) + .then(() => + awsRequest(auth, PutObjectCommand, { + Bucket: testBucket, + Body: testBody, + Key: testKey, + }), + )); + + it('should grant write access with grant-full-control', () => + s3 + .send( + new PutBucketAclCommand({ + Bucket: testBucket, + GrantFullControl: grantUri, + }), + ) + .then(() => + awsRequest(auth, PutObjectCommand, { + Bucket: testBucket, + Body: testBody, + Key: testKey, + }), + )); it('should not grant write access', done => { s3.send(new PutBucketAclCommand(aclParam)) - .then(() => awsRequest(auth, PutObjectCommand, { - Bucket: testBucket, - Body: testBody, - Key: testKey, - })) + .then(() => + awsRequest(auth, PutObjectCommand, { + Bucket: testBucket, + Body: testBody, + Key: testKey, + }), + ) .then(() => done(new Error('Expected failure'))) .catch(cbWithError(done)); }); - itSkipIfE2E('should grant write access on an object not owned by the grantee', - () => s3.send(new PutBucketAclCommand({ - Bucket: testBucket, - GrantWrite: grantUri, - })) - .then(() => awsRequest(auth, PutObjectCommand, { - Bucket: testBucket, - Body: testBody, - Key: ownerObjKey, - }))); + itSkipIfE2E('should grant write access on an object not owned by the grantee', () => + s3 + .send( + new PutBucketAclCommand({ + Bucket: testBucket, + GrantWrite: grantUri, + }), + ) + .then(() => + awsRequest(auth, PutObjectCommand, { + Bucket: testBucket, + Body: testBody, + Key: ownerObjKey, + }), + ), + ); it(`should ${auth ? '' : 'not '}delete object not owned by the grantee`, done => { - s3.send(new PutBucketAclCommand({ - Bucket: testBucket, - GrantWrite: grantUri, - })) - .then(() => awsRequest(auth, DeleteObjectCommand, { + s3.send( + new PutBucketAclCommand({ Bucket: testBucket, - Key: ownerObjKey, - })) + GrantWrite: grantUri, + }), + ) + .then(() => + awsRequest(auth, DeleteObjectCommand, { + Bucket: testBucket, + Key: ownerObjKey, + }), + ) .then(() => { if (auth) { done(); @@ -210,16 +243,24 @@ withV4(sigCfg => { }); }); - it('should read bucket acl', () => s3.send(new PutBucketAclCommand({ - Bucket: testBucket, - GrantReadACP: grantUri, - })) + it('should read bucket acl', () => + s3 + .send( + new PutBucketAclCommand({ + Bucket: testBucket, + GrantReadACP: grantUri, + }), + ) .then(() => awsRequest(auth, GetBucketAclCommand, { Bucket: testBucket }))); - it('should read bucket acl with grant-full-control', () => s3.send(new PutBucketAclCommand({ - Bucket: testBucket, - GrantFullControl: grantUri, - })) + it('should read bucket acl with grant-full-control', () => + s3 + .send( + new PutBucketAclCommand({ + Bucket: testBucket, + GrantFullControl: grantUri, + }), + ) .then(() => awsRequest(auth, GetBucketAclCommand, { Bucket: testBucket }))); it('should not read bucket acl', done => { @@ -229,30 +270,44 @@ withV4(sigCfg => { .catch(cbWithError(done)); }); - it('should write bucket acl', () => s3.send(new PutBucketAclCommand({ - Bucket: testBucket, - GrantWriteACP: grantUri, - })) - .then(() => awsRequest(auth, PutBucketAclCommand, { - Bucket: testBucket, - GrantReadACP: `uri=${constants.publicId}`, - }))); - - it('should write bucket acl with grant-full-control', () => s3.send(new PutBucketAclCommand({ - Bucket: testBucket, - GrantFullControl: grantUri, - })) - .then(() => awsRequest(auth, PutBucketAclCommand, { - Bucket: testBucket, - GrantReadACP: `uri=${constants.publicId}`, - }))); + it('should write bucket acl', () => + s3 + .send( + new PutBucketAclCommand({ + Bucket: testBucket, + GrantWriteACP: grantUri, + }), + ) + .then(() => + awsRequest(auth, PutBucketAclCommand, { + Bucket: testBucket, + GrantReadACP: `uri=${constants.publicId}`, + }), + )); + + it('should write bucket acl with grant-full-control', () => + s3 + .send( + new PutBucketAclCommand({ + Bucket: testBucket, + GrantFullControl: grantUri, + }), + ) + .then(() => + awsRequest(auth, PutBucketAclCommand, { + Bucket: testBucket, + GrantReadACP: `uri=${constants.publicId}`, + }), + )); it('should not write bucket acl', done => { s3.send(new PutBucketAclCommand(aclParam)) - .then(() => awsRequest(auth, PutBucketAclCommand, { - Bucket: testBucket, - GrantReadACP: `uri=${constants.allAuthedUsersId}`, - })) + .then(() => + awsRequest(auth, PutBucketAclCommand, { + Bucket: testBucket, + GrantReadACP: `uri=${constants.allAuthedUsersId}`, + }), + ) .then(() => done(new Error('Expected failure'))) .catch(cbWithError(done)); }); @@ -267,11 +322,13 @@ withV4(sigCfg => { beforeEach(async () => { await s3.send(new CreateBucketCommand({ Bucket: testBucket })); - await s3.send(new PutObjectCommand({ - Bucket: testBucket, - Body: testBody, - Key: testKey, - })); + await s3.send( + new PutObjectCommand({ + Bucket: testBucket, + Body: testBody, + Key: testKey, + }), + ); }); afterEach(async () => { @@ -279,95 +336,137 @@ withV4(sigCfg => { await ownerAccountBucketUtil.deleteOne(testBucket); }); - it('should grant read access', () => s3.send(new PutObjectAclCommand({ - Bucket: testBucket, - GrantRead: grantUri, - Key: testKey, - })) - .then(() => awsRequest(auth, GetObjectCommand, { - Bucket: testBucket, - Key: testKey, - }))); - - it('should grant read access with grant-full-control', () => s3.send(new PutObjectAclCommand({ - Bucket: testBucket, - GrantFullControl: grantUri, - Key: testKey, - })) - .then(() => awsRequest(auth, GetObjectCommand, { - Bucket: testBucket, - Key: testKey, - }))); + it('should grant read access', () => + s3 + .send( + new PutObjectAclCommand({ + Bucket: testBucket, + GrantRead: grantUri, + Key: testKey, + }), + ) + .then(() => + awsRequest(auth, GetObjectCommand, { + Bucket: testBucket, + Key: testKey, + }), + )); + + it('should grant read access with grant-full-control', () => + s3 + .send( + new PutObjectAclCommand({ + Bucket: testBucket, + GrantFullControl: grantUri, + Key: testKey, + }), + ) + .then(() => + awsRequest(auth, GetObjectCommand, { + Bucket: testBucket, + Key: testKey, + }), + )); it('should not grant read access', done => { s3.send(new PutObjectAclCommand(aclParam)) - .then(() => awsRequest(auth, GetObjectCommand, { - Bucket: testBucket, - Key: testKey, - })) + .then(() => + awsRequest(auth, GetObjectCommand, { + Bucket: testBucket, + Key: testKey, + }), + ) .then(() => done(new Error('Expected failure'))) .catch(cbWithError(done)); }); - it('should read object acl', () => s3.send(new PutObjectAclCommand({ - Bucket: testBucket, - GrantReadACP: grantUri, - Key: testKey, - })) - .then(() => awsRequest(auth, GetObjectAclCommand, { - Bucket: testBucket, - Key: testKey, - }))); - - it('should read object acl with grant-full-control', () => s3.send(new PutObjectAclCommand({ - Bucket: testBucket, - GrantFullControl: grantUri, - Key: testKey, - })) - .then(() => awsRequest(auth, GetObjectAclCommand, { - Bucket: testBucket, - Key: testKey, - }))); + it('should read object acl', () => + s3 + .send( + new PutObjectAclCommand({ + Bucket: testBucket, + GrantReadACP: grantUri, + Key: testKey, + }), + ) + .then(() => + awsRequest(auth, GetObjectAclCommand, { + Bucket: testBucket, + Key: testKey, + }), + )); + + it('should read object acl with grant-full-control', () => + s3 + .send( + new PutObjectAclCommand({ + Bucket: testBucket, + GrantFullControl: grantUri, + Key: testKey, + }), + ) + .then(() => + awsRequest(auth, GetObjectAclCommand, { + Bucket: testBucket, + Key: testKey, + }), + )); it('should not read object acl', done => { s3.send(new PutObjectAclCommand(aclParam)) - .then(() => awsRequest(auth, GetObjectAclCommand, { - Bucket: testBucket, - Key: testKey, - })) + .then(() => + awsRequest(auth, GetObjectAclCommand, { + Bucket: testBucket, + Key: testKey, + }), + ) .then(() => done(new Error('Expected failure'))) .catch(cbWithError(done)); }); - it('should write object acl', () => s3.send(new PutObjectAclCommand({ - Bucket: testBucket, - GrantWriteACP: grantUri, - Key: testKey, - })) - .then(() => awsRequest(auth, PutObjectAclCommand, { - Bucket: testBucket, - Key: testKey, - GrantReadACP: grantUri, - }))); - - it('should write object acl with grant-full-control', () => s3.send(new PutObjectAclCommand({ - Bucket: testBucket, - GrantFullControl: grantUri, - Key: testKey, - })) - .then(() => awsRequest(auth, PutObjectAclCommand, { - Bucket: testBucket, - Key: testKey, - GrantReadACP: `uri=${constants.publicId}`, - }))); + it('should write object acl', () => + s3 + .send( + new PutObjectAclCommand({ + Bucket: testBucket, + GrantWriteACP: grantUri, + Key: testKey, + }), + ) + .then(() => + awsRequest(auth, PutObjectAclCommand, { + Bucket: testBucket, + Key: testKey, + GrantReadACP: grantUri, + }), + )); + + it('should write object acl with grant-full-control', () => + s3 + .send( + new PutObjectAclCommand({ + Bucket: testBucket, + GrantFullControl: grantUri, + Key: testKey, + }), + ) + .then(() => + awsRequest(auth, PutObjectAclCommand, { + Bucket: testBucket, + Key: testKey, + GrantReadACP: `uri=${constants.publicId}`, + }), + )); it('should not write object acl', done => { s3.send(new PutObjectAclCommand(aclParam)) - .then(() => awsRequest(auth, PutObjectAclCommand, { - Bucket: testBucket, - Key: testKey, - GrantReadACP: `uri=${constants.allAuthedUsersId}`, - })) + .then(() => + awsRequest(auth, PutObjectAclCommand, { + Bucket: testBucket, + Key: testKey, + GrantReadACP: `uri=${constants.allAuthedUsersId}`, + }), + ) .then(() => done(new Error('Expected failure'))) .catch(cbWithError(done)); }); diff --git a/tests/functional/aws-node-sdk/test/bucket/bucketPolicyBypassPort.js b/tests/functional/aws-node-sdk/test/bucket/bucketPolicyBypassPort.js index cea6eff071..38b72c782f 100644 --- a/tests/functional/aws-node-sdk/test/bucket/bucketPolicyBypassPort.js +++ b/tests/functional/aws-node-sdk/test/bucket/bucketPolicyBypassPort.js @@ -1,15 +1,15 @@ const assert = require('assert'); -const { - S3Client, - PutObjectCommand, +const { + S3Client, + PutObjectCommand, GetObjectCommand, PutBucketPolicyCommand, DeleteBucketPolicyCommand, } = require('@aws-sdk/client-s3'); -const { - IAMClient, - CreatePolicyCommand, - CreateUserCommand, +const { + IAMClient, + CreatePolicyCommand, + CreateUserCommand, AttachUserPolicyCommand, DetachUserPolicyCommand, DeletePolicyCommand, @@ -18,12 +18,9 @@ const { CreateRoleCommand, AttachRolePolicyCommand, DetachRolePolicyCommand, - DeleteRoleCommand + DeleteRoleCommand, } = require('@aws-sdk/client-iam'); -const { - STSClient, - AssumeRoleCommand -} = require('@aws-sdk/client-sts'); +const { STSClient, AssumeRoleCommand } = require('@aws-sdk/client-sts'); const { v4: uuid } = require('uuid'); const getConfig = require('../support/config'); @@ -55,50 +52,58 @@ describeBypass('Bucket Policy Bypass Port', () => { before(async () => { await bucketUtilAccount.createOne(bucketName); - await s3ClientAccount.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectKey, - Body: objectContent, - })); + await s3ClientAccount.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectKey, + Body: objectContent, + }), + ); - await s3ClientAccount.send(new PutBucketPolicyCommand({ - Bucket: bucketName, - Policy: JSON.stringify({ - Version: '2012-10-17', - Statement: [ - { - Sid: 'DenyAllAccess', - Effect: 'Deny', - Principal: '*', - Action: 's3:*', - Resource: [`arn:aws:s3:::${bucketName}`, `arn:aws:s3:::${bucketName}/*`], - }, - ], + await s3ClientAccount.send( + new PutBucketPolicyCommand({ + Bucket: bucketName, + Policy: JSON.stringify({ + Version: '2012-10-17', + Statement: [ + { + Sid: 'DenyAllAccess', + Effect: 'Deny', + Principal: '*', + Action: 's3:*', + Resource: [`arn:aws:s3:::${bucketName}`, `arn:aws:s3:::${bucketName}/*`], + }, + ], + }), }), - })); + ); // create iam policy allow all actions for user and role - const policyRes = await iamClient.send(new CreatePolicyCommand({ - PolicyName: 'bp-bypass-policy', - PolicyDocument: JSON.stringify({ - Version: '2012-10-17', - Statement: [ - { - Sid: 'AllowAllActions', - Effect: 'Allow', - Action: '*', - Resource: ['*'], - }, - ], + const policyRes = await iamClient.send( + new CreatePolicyCommand({ + PolicyName: 'bp-bypass-policy', + PolicyDocument: JSON.stringify({ + Version: '2012-10-17', + Statement: [ + { + Sid: 'AllowAllActions', + Effect: 'Allow', + Action: '*', + Resource: ['*'], + }, + ], + }), }), - })); + ); policyAllowAllActions = policyRes.Policy; await iamClient.send(new CreateUserCommand({ UserName: userName })); - await iamClient.send(new AttachUserPolicyCommand({ - UserName: userName, - PolicyArn: policyAllowAllActions.Arn, - })); + await iamClient.send( + new AttachUserPolicyCommand({ + UserName: userName, + PolicyArn: policyAllowAllActions.Arn, + }), + ); }); after(async () => { @@ -108,20 +113,24 @@ describeBypass('Bucket Policy Bypass Port', () => { await bucketUtilAccount.deleteOne(bucketName); if (policyAllowAllActions) { - await iamClient.send(new DetachUserPolicyCommand({ - UserName: userName, - PolicyArn: policyAllowAllActions.Arn, - })); + await iamClient.send( + new DetachUserPolicyCommand({ + UserName: userName, + PolicyArn: policyAllowAllActions.Arn, + }), + ); await iamClient.send(new DeletePolicyCommand({ PolicyArn: policyAllowAllActions.Arn })); } await iamClient.send(new DeleteUserCommand({ UserName: userName })); }); it('should allow account root access on s3 port', async () => { - const getResponse = await s3ClientAccount.send(new GetObjectCommand({ - Bucket: bucketName, - Key: objectKey, - })); + const getResponse = await s3ClientAccount.send( + new GetObjectCommand({ + Bucket: bucketName, + Key: objectKey, + }), + ); assert(getResponse.Body, 'Should be able to get object'); const bodyString = await getResponse.Body.transformToString(); assert.strictEqual(bodyString, objectContent); @@ -157,10 +166,12 @@ describeBypass('Bucket Policy Bypass Port', () => { it('should deny user access on s3 port', async () => { try { - await userS3Client.send(new GetObjectCommand({ - Bucket: bucketName, - Key: objectKey, - })); + await userS3Client.send( + new GetObjectCommand({ + Bucket: bucketName, + Key: objectKey, + }), + ); assert.fail('Expected AccessDenied error for getObject'); } catch (err) { assert.strictEqual(err.name, 'AccessDenied'); @@ -168,10 +179,12 @@ describeBypass('Bucket Policy Bypass Port', () => { }); it('should bypass user bucket policy on internal port', async () => { - const getResponse = await userInternalBypassBPS3Client.send(new GetObjectCommand({ - Bucket: bucketName, - Key: objectKey, - })); + const getResponse = await userInternalBypassBPS3Client.send( + new GetObjectCommand({ + Bucket: bucketName, + Key: objectKey, + }), + ); assert(getResponse.Body, 'Should be able to get object on internal port'); const bodyString = await getResponse.Body.transformToString(); assert.strictEqual(bodyString, objectContent); @@ -184,24 +197,28 @@ describeBypass('Bucket Policy Bypass Port', () => { let stsClient; before(async () => { - const roleRes = await iamClient.send(new CreateRoleCommand({ - RoleName: roleName, - AssumeRolePolicyDocument: JSON.stringify({ - Version: '2012-10-17', - Statement: [ - { - Effect: 'Allow', - Principal: '*', - Action: 'sts:AssumeRole', - }, - ], + const roleRes = await iamClient.send( + new CreateRoleCommand({ + RoleName: roleName, + AssumeRolePolicyDocument: JSON.stringify({ + Version: '2012-10-17', + Statement: [ + { + Effect: 'Allow', + Principal: '*', + Action: 'sts:AssumeRole', + }, + ], + }), }), - })); + ); - await iamClient.send(new AttachRolePolicyCommand({ - RoleName: roleName, - PolicyArn: policyAllowAllActions.Arn, - })); + await iamClient.send( + new AttachRolePolicyCommand({ + RoleName: roleName, + PolicyArn: policyAllowAllActions.Arn, + }), + ); const accessKeyResponse = await iamClient.send(new CreateAccessKeyCommand({ UserName: userName })); @@ -216,10 +233,12 @@ describeBypass('Bucket Policy Bypass Port', () => { stsClient = new STSClient(stsConfig); // Assume role to get temporary credentials - const assumeRoleResponse = await stsClient.send(new AssumeRoleCommand({ - RoleArn: roleRes.Role.Arn, - RoleSessionName: 'bp-bypass-session', - })); + const assumeRoleResponse = await stsClient.send( + new AssumeRoleCommand({ + RoleArn: roleRes.Role.Arn, + RoleSessionName: 'bp-bypass-session', + }), + ); const credentials = assumeRoleResponse.Credentials; // Create S3 client for role (regular port) @@ -245,19 +264,23 @@ describeBypass('Bucket Policy Bypass Port', () => { }); after(async () => { - await iamClient.send(new DetachRolePolicyCommand({ - RoleName: roleName, - PolicyArn: policyAllowAllActions.Arn, - })); + await iamClient.send( + new DetachRolePolicyCommand({ + RoleName: roleName, + PolicyArn: policyAllowAllActions.Arn, + }), + ); await iamClient.send(new DeleteRoleCommand({ RoleName: roleName })); }); it('should deny role access on s3 port', async () => { try { - await roleS3Client.send(new GetObjectCommand({ - Bucket: bucketName, - Key: objectKey, - })); + await roleS3Client.send( + new GetObjectCommand({ + Bucket: bucketName, + Key: objectKey, + }), + ); assert.fail('Expected AccessDenied error for getObject'); } catch (err) { assert.strictEqual(err.name, 'AccessDenied'); @@ -265,10 +288,12 @@ describeBypass('Bucket Policy Bypass Port', () => { }); it('should bypass role bucket policy on internal port', async () => { - const getResponse = await roleInternalBypassBPS3Client.send(new GetObjectCommand({ - Bucket: bucketName, - Key: objectKey, - })); + const getResponse = await roleInternalBypassBPS3Client.send( + new GetObjectCommand({ + Bucket: bucketName, + Key: objectKey, + }), + ); assert(getResponse.Body, 'Should be able to get object on internal port'); const bodyString = await getResponse.Body.transformToString(); assert.strictEqual(bodyString, objectContent); diff --git a/tests/functional/aws-node-sdk/test/bucket/bucketPolicyWithResourceStatements.js b/tests/functional/aws-node-sdk/test/bucket/bucketPolicyWithResourceStatements.js index b45d1e4209..984e54138f 100644 --- a/tests/functional/aws-node-sdk/test/bucket/bucketPolicyWithResourceStatements.js +++ b/tests/functional/aws-node-sdk/test/bucket/bucketPolicyWithResourceStatements.js @@ -3,7 +3,8 @@ const { PutBucketPolicyCommand, ListObjectsCommand, GetObjectCommand, - PutObjectCommand } = require('@aws-sdk/client-s3'); + PutObjectCommand, +} = require('@aws-sdk/client-s3'); const { errorInstances } = require('arsenal'); const withV4 = require('../support/withV4'); @@ -18,12 +19,13 @@ withV4(sigCfg => { if (auth) { // Use authenticated client const commandMap = { - 'listObjects': ListObjectsCommand, - 'getObject': GetObjectCommand, - 'putObject': PutObjectCommand, + listObjects: ListObjectsCommand, + getObject: GetObjectCommand, + putObject: PutObjectCommand, }; const CommandCtor = commandMap[operation]; - ownerAccountBucketUtil.s3.send(new CommandCtor(params)) + ownerAccountBucketUtil.s3 + .send(new CommandCtor(params)) .then(data => callback(null, data)) .catch(err => callback(err)); } else { @@ -35,12 +37,13 @@ withV4(sigCfg => { signer: { sign: async request => request }, }); const commandMap = { - 'listObjects': ListObjectsCommand, - 'getObject': GetObjectCommand, - 'putObject': PutObjectCommand, + listObjects: ListObjectsCommand, + getObject: GetObjectCommand, + putObject: PutObjectCommand, }; const CommandCtor = commandMap[operation]; - unauthClient.s3.send(new CommandCtor(params)) + unauthClient.s3 + .send(new CommandCtor(params)) .then(data => callback(null, data)) .catch(err => callback(err)); } @@ -62,8 +65,9 @@ withV4(sigCfg => { describe('Bucket policies with resource statement', () => { beforeEach(() => ownerAccountBucketUtil.createMany(testBuckets)); - afterEach(() => ownerAccountBucketUtil.emptyMany(testBuckets) - .then(() => ownerAccountBucketUtil.deleteMany(testBuckets))); + afterEach(() => + ownerAccountBucketUtil.emptyMany(testBuckets).then(() => ownerAccountBucketUtil.deleteMany(testBuckets)), + ); it('should allow action on a bucket specified in the policy', done => { const statement = { @@ -77,11 +81,12 @@ withV4(sigCfg => { Version: '2012-10-17', Statement: [statement], }; - s3.send(new PutBucketPolicyCommand({ - Bucket: testBuckets[0], - Policy: JSON.stringify(bucketPolicy), - })) - .then(() => { + s3.send( + new PutBucketPolicyCommand({ + Bucket: testBuckets[0], + Policy: JSON.stringify(bucketPolicy), + }), + ).then(() => { const param = { Bucket: testBuckets[0] }; awsRequest(true, 'listObjects', param, cbNoError(done)); }); @@ -99,11 +104,12 @@ withV4(sigCfg => { Version: '2012-10-17', Statement: [statement], }; - s3.send(new PutBucketPolicyCommand({ - Bucket: testBuckets[0], - Policy: JSON.stringify(bucketPolicy), - })) - .then(() => { + s3.send( + new PutBucketPolicyCommand({ + Bucket: testBuckets[0], + Policy: JSON.stringify(bucketPolicy), + }), + ).then(() => { const param = { Bucket: testBuckets[1] }; awsRequest(false, 'listObjects', param, cbWithError(done)); }); @@ -121,11 +127,12 @@ withV4(sigCfg => { Version: '2012-10-17', Statement: [statement], }; - s3.send(new PutBucketPolicyCommand({ - Bucket: testBuckets[0], - Policy: JSON.stringify(bucketPolicy), - })) - .then(() => { + s3.send( + new PutBucketPolicyCommand({ + Bucket: testBuckets[0], + Policy: JSON.stringify(bucketPolicy), + }), + ).then(() => { const param = { Bucket: testBuckets[0] }; awsRequest(false, 'listObjects', param, cbWithError(done)); }); @@ -145,22 +152,28 @@ withV4(sigCfg => { Version: '2012-10-17', Statement: [statement], }; - s3.send(new PutBucketPolicyCommand({ - Bucket: testBuckets[0], - Policy: JSON.stringify(bucketPolicy), - })) - .then(() => s3.send(new PutObjectCommand({ - Bucket: testBuckets[0], - Body: testBody, - Key: testKey, - }))) - .then(() => { - const param = { + s3.send( + new PutBucketPolicyCommand({ Bucket: testBuckets[0], - Key: testKey, - }; - awsRequest(false, 'getObject', param, cbNoError(done)); - }); + Policy: JSON.stringify(bucketPolicy), + }), + ) + .then(() => + s3.send( + new PutObjectCommand({ + Bucket: testBuckets[0], + Body: testBody, + Key: testKey, + }), + ), + ) + .then(() => { + const param = { + Bucket: testBuckets[0], + Key: testKey, + }; + awsRequest(false, 'getObject', param, cbNoError(done)); + }); }); it('should allow action on an object satisfying the wildcard in the policy', done => { @@ -177,22 +190,28 @@ withV4(sigCfg => { Version: '2012-10-17', Statement: [statement], }; - s3.send(new PutBucketPolicyCommand({ - Bucket: testBuckets[0], - Policy: JSON.stringify(bucketPolicy), - })) - .then(() => s3.send(new PutObjectCommand({ - Bucket: testBuckets[0], - Body: testBody, - Key: testKey, - }))) - .then(() => { - const param = { + s3.send( + new PutBucketPolicyCommand({ Bucket: testBuckets[0], - Key: testKey, - }; - awsRequest(false, 'getObject', param, cbNoError(done)); - }); + Policy: JSON.stringify(bucketPolicy), + }), + ) + .then(() => + s3.send( + new PutObjectCommand({ + Bucket: testBuckets[0], + Body: testBody, + Key: testKey, + }), + ), + ) + .then(() => { + const param = { + Bucket: testBuckets[0], + Key: testKey, + }; + awsRequest(false, 'getObject', param, cbNoError(done)); + }); }); it('should deny action on an object specified in the policy', done => { @@ -209,22 +228,28 @@ withV4(sigCfg => { Version: '2012-10-17', Statement: [statement], }; - s3.send(new PutBucketPolicyCommand({ - Bucket: testBuckets[0], - Policy: JSON.stringify(bucketPolicy), - })) - .then(() => s3.send(new PutObjectCommand({ + s3.send( + new PutBucketPolicyCommand({ Bucket: testBuckets[0], - Body: testBody, - Key: testKey, - }))) - .then(() => { - const param = { - Bucket: testBuckets[0], - Key: testKey, - }; - awsRequest(false, 'getObject', param, cbWithError(done)); - }); + Policy: JSON.stringify(bucketPolicy), + }), + ) + .then(() => + s3.send( + new PutObjectCommand({ + Bucket: testBuckets[0], + Body: testBody, + Key: testKey, + }), + ), + ) + .then(() => { + const param = { + Bucket: testBuckets[0], + Key: testKey, + }; + awsRequest(false, 'getObject', param, cbWithError(done)); + }); }); it('should deny action on an object not specified in the policy', done => { @@ -240,11 +265,12 @@ withV4(sigCfg => { Version: '2012-10-17', Statement: [statement], }; - s3.send(new PutBucketPolicyCommand({ - Bucket: testBuckets[0], - Policy: JSON.stringify(bucketPolicy), - })) - .then(() => { + s3.send( + new PutBucketPolicyCommand({ + Bucket: testBuckets[0], + Policy: JSON.stringify(bucketPolicy), + }), + ).then(() => { const param = { Bucket: testBuckets[0], Key: 'invalidkey', @@ -266,11 +292,12 @@ withV4(sigCfg => { Version: '2012-10-17', Statement: [statement], }; - s3.send(new PutBucketPolicyCommand({ - Bucket: testBuckets[0], - Policy: JSON.stringify(bucketPolicy), - })) - .then(() => { + s3.send( + new PutBucketPolicyCommand({ + Bucket: testBuckets[0], + Policy: JSON.stringify(bucketPolicy), + }), + ).then(() => { const param = { Bucket: testBuckets[1], Key: 'invalidkey', diff --git a/tests/functional/aws-node-sdk/test/bucket/deleteBucketLifecycle.js b/tests/functional/aws-node-sdk/test/bucket/deleteBucketLifecycle.js index d454c15752..6f5e8ebd40 100644 --- a/tests/functional/aws-node-sdk/test/bucket/deleteBucketLifecycle.js +++ b/tests/functional/aws-node-sdk/test/bucket/deleteBucketLifecycle.js @@ -1,11 +1,13 @@ const assert = require('assert'); const { errors } = require('arsenal'); -const { S3Client, +const { + S3Client, CreateBucketCommand, DeleteBucketCommand, DeleteBucketLifecycleCommand, PutBucketLifecycleConfigurationCommand, - GetBucketLifecycleConfigurationCommand } = require('@aws-sdk/client-s3'); + GetBucketLifecycleConfigurationCommand, +} = require('@aws-sdk/client-s3'); const getConfig = require('../support/config'); @@ -24,11 +26,16 @@ function assertError(err, expectedErr) { if (expectedErr === null) { assert.strictEqual(err, null, `expected no error but got '${err}'`); } else { - assert.strictEqual(err.name, expectedErr, 'incorrect error response ' + - `code: should be '${expectedErr}' but got '${err.Code}'`); - assert.strictEqual(err.$metadata.httpStatusCode, errors[expectedErr].code, - 'incorrect error status code: should be 400 but got ' + - `'${err.$metadata.httpStatusCode}'`); + assert.strictEqual( + err.name, + expectedErr, + 'incorrect error response ' + `code: should be '${expectedErr}' but got '${err.Code}'`, + ); + assert.strictEqual( + err.$metadata.httpStatusCode, + errors[expectedErr].code, + 'incorrect error status code: should be 400 but got ' + `'${err.$metadata.httpStatusCode}'`, + ); } } @@ -67,12 +74,11 @@ describe('aws-sdk test delete bucket lifecycle', () => { } }); - it('should return no error if no lifecycle config on bucket', () => s3.send(new - DeleteBucketLifecycleCommand({ Bucket: bucket }))); + it('should return no error if no lifecycle config on bucket', () => + s3.send(new DeleteBucketLifecycleCommand({ Bucket: bucket }))); it('should delete lifecycle configuration from bucket', async () => { - const params = { Bucket: bucket, - LifecycleConfiguration: { Rules: [basicRule] } }; + const params = { Bucket: bucket, LifecycleConfiguration: { Rules: [basicRule] } }; await s3.send(new PutBucketLifecycleConfigurationCommand(params)); await s3.send(new DeleteBucketLifecycleCommand({ Bucket: bucket })); try { diff --git a/tests/functional/aws-node-sdk/test/bucket/deleteBucketPolicy.js b/tests/functional/aws-node-sdk/test/bucket/deleteBucketPolicy.js index 3276283c3d..97ad6c4e89 100644 --- a/tests/functional/aws-node-sdk/test/bucket/deleteBucketPolicy.js +++ b/tests/functional/aws-node-sdk/test/bucket/deleteBucketPolicy.js @@ -1,11 +1,13 @@ const assert = require('assert'); const { errors } = require('arsenal'); -const { S3Client, +const { + S3Client, CreateBucketCommand, DeleteBucketCommand, DeleteBucketPolicyCommand, PutBucketPolicyCommand, - GetBucketPolicyCommand } = require('@aws-sdk/client-s3'); + GetBucketPolicyCommand, +} = require('@aws-sdk/client-s3'); const getConfig = require('../support/config'); const BucketUtility = require('../../lib/utility/bucket-util'); @@ -13,13 +15,15 @@ const BucketUtility = require('../../lib/utility/bucket-util'); const bucket = 'deletebucketpolicy-test-bucket'; const bucketPolicy = { Version: '2012-10-17', - Statement: [{ - Sid: 'testid', - Effect: 'Allow', - Principal: '*', - Action: 's3:putBucketPolicy', - Resource: `arn:aws:s3:::${bucket}`, - }], + Statement: [ + { + Sid: 'testid', + Effect: 'Allow', + Principal: '*', + Action: 's3:putBucketPolicy', + Resource: `arn:aws:s3:::${bucket}`, + }, + ], }; // Check for the expected error response code and status code. @@ -27,11 +31,16 @@ function assertError(err, expectedErr) { if (expectedErr === null) { assert.strictEqual(err, null, `expected no error but got '${err}'`); } else { - assert.strictEqual(err.name, expectedErr, 'incorrect error response ' + - `code: should be '${expectedErr}' but got '${err.name}'`); - assert.strictEqual(err.$metadata.httpStatusCode, errors[expectedErr].code, - 'incorrect error status code: should be 400 but got ' + - `'${err.$metadata.httpStatusCode}'`); + assert.strictEqual( + err.name, + expectedErr, + 'incorrect error response ' + `code: should be '${expectedErr}' but got '${err.name}'`, + ); + assert.strictEqual( + err.$metadata.httpStatusCode, + errors[expectedErr].code, + 'incorrect error status code: should be 400 but got ' + `'${err.$metadata.httpStatusCode}'`, + ); } } diff --git a/tests/functional/aws-node-sdk/test/bucket/deleteBucketQuota.js b/tests/functional/aws-node-sdk/test/bucket/deleteBucketQuota.js index 0e58a7b901..e1f1aa723b 100644 --- a/tests/functional/aws-node-sdk/test/bucket/deleteBucketQuota.js +++ b/tests/functional/aws-node-sdk/test/bucket/deleteBucketQuota.js @@ -1,6 +1,4 @@ -const { S3Client, - CreateBucketCommand, - DeleteBucketCommand } = require('@aws-sdk/client-s3'); +const { S3Client, CreateBucketCommand, DeleteBucketCommand } = require('@aws-sdk/client-s3'); const assert = require('assert'); const getConfig = require('../support/config'); const sendRequest = require('../quota/tooling').sendRequest; diff --git a/tests/functional/aws-node-sdk/test/bucket/deleteBucketRateLimit.js b/tests/functional/aws-node-sdk/test/bucket/deleteBucketRateLimit.js index d232f6d206..3cde37edde 100644 --- a/tests/functional/aws-node-sdk/test/bucket/deleteBucketRateLimit.js +++ b/tests/functional/aws-node-sdk/test/bucket/deleteBucketRateLimit.js @@ -1,9 +1,5 @@ const assert = require('assert'); -const { - S3Client, - CreateBucketCommand, - DeleteBucketCommand, -} = require('@aws-sdk/client-s3'); +const { S3Client, CreateBucketCommand, DeleteBucketCommand } = require('@aws-sdk/client-s3'); const getConfig = require('../support/config'); const { sendRateLimitRequest, skipIfRateLimitDisabled } = require('../rateLimit/tooling'); @@ -36,17 +32,19 @@ skipIfRateLimitDisabled('Test delete bucket rate limit', () => { try { // First set a rate limit config const rateLimitConfig = { RequestsPerSecond: 150 }; - await sendRateLimitRequest('PUT', '127.0.0.1:8000', - `/${bucket}/?rate-limit`, JSON.stringify(rateLimitConfig)); + await sendRateLimitRequest( + 'PUT', + '127.0.0.1:8000', + `/${bucket}/?rate-limit`, + JSON.stringify(rateLimitConfig), + ); // Then delete it - await sendRateLimitRequest('DELETE', '127.0.0.1:8000', - `/${bucket}/?rate-limit`); + await sendRateLimitRequest('DELETE', '127.0.0.1:8000', `/${bucket}/?rate-limit`); // Verify it's deleted try { - await sendRateLimitRequest('GET', '127.0.0.1:8000', - `/${bucket}/?rate-limit`); + await sendRateLimitRequest('GET', '127.0.0.1:8000', `/${bucket}/?rate-limit`); assert.fail('Expected NoSuchRateLimitConfig error'); } catch (err) { assert.strictEqual(err.Error.Code[0], 'NoSuchRateLimitConfig'); @@ -58,8 +56,7 @@ skipIfRateLimitDisabled('Test delete bucket rate limit', () => { it('should not return an error even if no rate limit config exists', async () => { try { - await sendRateLimitRequest('DELETE', '127.0.0.1:8000', - `/${bucket}/?rate-limit`); + await sendRateLimitRequest('DELETE', '127.0.0.1:8000', `/${bucket}/?rate-limit`); assert.ok(true); } catch (err) { assert.ifError(err); @@ -68,8 +65,7 @@ skipIfRateLimitDisabled('Test delete bucket rate limit', () => { it('should return NoSuchBucket error when bucket does not exist', async () => { try { - await sendRateLimitRequest('DELETE', '127.0.0.1:8000', - `/${nonExistentBucket}/?rate-limit`); + await sendRateLimitRequest('DELETE', '127.0.0.1:8000', `/${nonExistentBucket}/?rate-limit`); } catch (err) { assert.strictEqual(err.Error.Code[0], 'NoSuchBucket'); } diff --git a/tests/functional/aws-node-sdk/test/bucket/deleteBucketReplication.js b/tests/functional/aws-node-sdk/test/bucket/deleteBucketReplication.js index ee3212aa04..a4bea67e04 100644 --- a/tests/functional/aws-node-sdk/test/bucket/deleteBucketReplication.js +++ b/tests/functional/aws-node-sdk/test/bucket/deleteBucketReplication.js @@ -1,11 +1,13 @@ const assert = require('assert'); -const { S3Client, +const { + S3Client, CreateBucketCommand, DeleteBucketCommand, PutBucketVersioningCommand, PutBucketReplicationCommand, DeleteBucketReplicationCommand, - GetBucketReplicationCommand } = require('@aws-sdk/client-s3'); + GetBucketReplicationCommand, +} = require('@aws-sdk/client-s3'); const { errorInstances } = require('arsenal'); const getConfig = require('../support/config'); @@ -13,8 +15,7 @@ const BucketUtility = require('../../lib/utility/bucket-util'); const bucket = 'source-bucket'; const replicationConfig = { - Role: 'arn:aws:iam::account-id:role/src-resource,' + - 'arn:aws:iam::account-id:role/dest-resource', + Role: 'arn:aws:iam::account-id:role/src-resource,' + 'arn:aws:iam::account-id:role/dest-resource', Rules: [ { Destination: { Bucket: 'arn:aws:s3:::destination-bucket' }, @@ -31,24 +32,27 @@ describe('aws-node-sdk test deleteBucketReplication', () => { const config = getConfig('default', { signatureVersion: 'v4' }); function putVersioningOnBucket(bucket) { - return s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Enabled' }, - })); + return s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ); } function putReplicationOnBucket(bucket) { - return s3.send(new PutBucketReplicationCommand({ - Bucket: bucket, - ReplicationConfiguration: replicationConfig, - })); + return s3.send( + new PutBucketReplicationCommand({ + Bucket: bucket, + ReplicationConfiguration: replicationConfig, + }), + ); } function deleteReplicationAndCheckResponse(bucket) { - return s3.send(new DeleteBucketReplicationCommand({ Bucket: bucket })) - .then(data => { - assert.deepStrictEqual(data.$metadata.httpStatusCode, 204); - }); + return s3.send(new DeleteBucketReplicationCommand({ Bucket: bucket })).then(data => { + assert.deepStrictEqual(data.$metadata.httpStatusCode, 204); + }); } beforeEach(() => { @@ -59,8 +63,8 @@ describe('aws-node-sdk test deleteBucketReplication', () => { afterEach(() => s3.send(new DeleteBucketCommand({ Bucket: bucket }))); - it('should return empty object if bucket has no replication config', - () => deleteReplicationAndCheckResponse(bucket)); + it('should return empty object if bucket has no replication config', () => + deleteReplicationAndCheckResponse(bucket)); it('should delete a bucket replication config when it has one', async () => { await putVersioningOnBucket(bucket); @@ -68,23 +72,26 @@ describe('aws-node-sdk test deleteBucketReplication', () => { await deleteReplicationAndCheckResponse(bucket); }); - it('should return ReplicationConfigurationNotFoundError if getting ' + - 'replication config after it has been deleted', async () => { - await putVersioningOnBucket(bucket); - await putReplicationOnBucket(bucket); - - const data = await s3.send(new GetBucketReplicationCommand({ Bucket: bucket })); - assert.deepStrictEqual(data.ReplicationConfiguration, replicationConfig); + it( + 'should return ReplicationConfigurationNotFoundError if getting ' + + 'replication config after it has been deleted', + async () => { + await putVersioningOnBucket(bucket); + await putReplicationOnBucket(bucket); - await deleteReplicationAndCheckResponse(bucket); - - try { - await s3.send(new GetBucketReplicationCommand({ Bucket: bucket })); - assert.fail('Expected ReplicationConfigurationNotFoundError'); - } catch (err) { - assert(errorInstances.ReplicationConfigurationNotFoundError.is[err.name]); - } - }); + const data = await s3.send(new GetBucketReplicationCommand({ Bucket: bucket })); + assert.deepStrictEqual(data.ReplicationConfiguration, replicationConfig); + + await deleteReplicationAndCheckResponse(bucket); + + try { + await s3.send(new GetBucketReplicationCommand({ Bucket: bucket })); + assert.fail('Expected ReplicationConfigurationNotFoundError'); + } catch (err) { + assert(errorInstances.ReplicationConfigurationNotFoundError.is[err.name]); + } + }, + ); it('should return AccessDenied if user is not bucket owner', async () => { try { diff --git a/tests/functional/aws-node-sdk/test/bucket/deleteBucketTagging.js b/tests/functional/aws-node-sdk/test/bucket/deleteBucketTagging.js index 2eecf0dd7a..e770e4b1a8 100644 --- a/tests/functional/aws-node-sdk/test/bucket/deleteBucketTagging.js +++ b/tests/functional/aws-node-sdk/test/bucket/deleteBucketTagging.js @@ -1,10 +1,12 @@ const assert = require('assert'); -const { S3Client, +const { + S3Client, CreateBucketCommand, DeleteBucketCommand, PutBucketTaggingCommand, GetBucketTaggingCommand, - DeleteBucketTaggingCommand } = require('@aws-sdk/client-s3'); + DeleteBucketTaggingCommand, +} = require('@aws-sdk/client-s3'); const assertError = require('../../../../utilities/bucketTagging-util'); @@ -39,25 +41,33 @@ describe('aws-sdk test delete bucket tagging', () => { afterEach(() => s3.send(new DeleteBucketCommand({ Bucket: bucket }))); it('should delete tag', async () => { - await s3.send(new PutBucketTaggingCommand({ - AccountId: s3.AccountId, - Tagging: validTagging, - Bucket: bucket, - })); - const res = await s3.send(new GetBucketTaggingCommand({ - AccountId: s3.AccountId, - Bucket: bucket, - })); + await s3.send( + new PutBucketTaggingCommand({ + AccountId: s3.AccountId, + Tagging: validTagging, + Bucket: bucket, + }), + ); + const res = await s3.send( + new GetBucketTaggingCommand({ + AccountId: s3.AccountId, + Bucket: bucket, + }), + ); assert.deepStrictEqual(res.TagSet, validTagging.TagSet); - await s3.send(new DeleteBucketTaggingCommand({ - AccountId: s3.AccountId, - Bucket: bucket, - })); - try { - await s3.send(new GetBucketTaggingCommand({ + await s3.send( + new DeleteBucketTaggingCommand({ AccountId: s3.AccountId, Bucket: bucket, - })); + }), + ); + try { + await s3.send( + new GetBucketTaggingCommand({ + AccountId: s3.AccountId, + Bucket: bucket, + }), + ); throw new Error('Expected NoSuchTagSet error'); } catch (err) { assertError(err, 'NoSuchTagSet'); @@ -66,23 +76,29 @@ describe('aws-sdk test delete bucket tagging', () => { it('should make no change when deleting tags on bucket with no tags', async () => { try { - await s3.send(new GetBucketTaggingCommand({ - AccountId: s3.AccountId, - Bucket: bucket, - })); + await s3.send( + new GetBucketTaggingCommand({ + AccountId: s3.AccountId, + Bucket: bucket, + }), + ); throw new Error('Expected NoSuchTagSet error'); } catch (err) { assertError(err, 'NoSuchTagSet'); } - await s3.send(new DeleteBucketTaggingCommand({ - AccountId: s3.AccountId, - Bucket: bucket, - })); - try { - await s3.send(new GetBucketTaggingCommand({ + await s3.send( + new DeleteBucketTaggingCommand({ AccountId: s3.AccountId, Bucket: bucket, - })); + }), + ); + try { + await s3.send( + new GetBucketTaggingCommand({ + AccountId: s3.AccountId, + Bucket: bucket, + }), + ); throw new Error('Expected NoSuchTagSet error'); } catch (err) { assertError(err, 'NoSuchTagSet'); diff --git a/tests/functional/aws-node-sdk/test/bucket/deleteCors.js b/tests/functional/aws-node-sdk/test/bucket/deleteCors.js index 88196a5276..647d4d429b 100644 --- a/tests/functional/aws-node-sdk/test/bucket/deleteCors.js +++ b/tests/functional/aws-node-sdk/test/bucket/deleteCors.js @@ -1,26 +1,29 @@ const assert = require('assert'); -const { S3Client, +const { + S3Client, CreateBucketCommand, DeleteBucketCommand, DeleteBucketCorsCommand, PutBucketCorsCommand, - GetBucketCorsCommand } = require('@aws-sdk/client-s3'); + GetBucketCorsCommand, +} = require('@aws-sdk/client-s3'); const withV4 = require('../support/withV4'); const getConfig = require('../support/config'); const bucketName = 'testdeletecorsbucket'; -const sampleCors = { CORSRules: [ - { AllowedMethods: ['PUT', 'POST', 'DELETE'], - AllowedOrigins: ['http://www.example.com'], - AllowedHeaders: ['*'], - MaxAgeSeconds: 3000, - ExposeHeaders: ['x-amz-server-side-encryption'] }, - { AllowedMethods: ['GET'], - AllowedOrigins: ['*'], - AllowedHeaders: ['*'], - MaxAgeSeconds: 3000 }, -] }; +const sampleCors = { + CORSRules: [ + { + AllowedMethods: ['PUT', 'POST', 'DELETE'], + AllowedOrigins: ['http://www.example.com'], + AllowedHeaders: ['*'], + MaxAgeSeconds: 3000, + ExposeHeaders: ['x-amz-server-side-encryption'], + }, + { AllowedMethods: ['GET'], AllowedOrigins: ['*'], AllowedHeaders: ['*'], MaxAgeSeconds: 3000 }, + ], +}; const itSkipIfAWS = process.env.AWS_ON_AIR ? it.skip : it; @@ -49,30 +52,31 @@ describe('DELETE bucket cors', () => { describe('with existing bucket', () => { beforeEach(() => s3.send(new CreateBucketCommand({ Bucket: bucketName }))); - + afterEach(() => deleteBucket(s3, bucketName)); describe('without existing cors configuration', () => { it('should return a 204 response', async () => { const res = await s3.send(new DeleteBucketCorsCommand({ Bucket: bucketName })); const statusCode = res?.$metadata?.httpStatusCode; - assert.strictEqual(statusCode, 204, - `Found unexpected statusCode ${statusCode}`); + assert.strictEqual(statusCode, 204, `Found unexpected statusCode ${statusCode}`); }); }); describe('with existing cors configuration', () => { - beforeEach(() => s3.send(new PutBucketCorsCommand({ - Bucket: bucketName, - CORSConfiguration: sampleCors - }))); - + beforeEach(() => + s3.send( + new PutBucketCorsCommand({ + Bucket: bucketName, + CORSConfiguration: sampleCors, + }), + ), + ); it('should delete bucket configuration successfully', async () => { const res = await s3.send(new DeleteBucketCorsCommand({ Bucket: bucketName })); const statusCode = res?.$metadata?.httpStatusCode; - assert.strictEqual(statusCode, 204, - `Found unexpected statusCode ${statusCode}`); + assert.strictEqual(statusCode, 204, `Found unexpected statusCode ${statusCode}`); try { await s3.send(new GetBucketCorsCommand({ Bucket: bucketName })); throw new Error('Expected NoSuchCORSConfiguration error'); @@ -88,8 +92,7 @@ describe('DELETE bucket cors', () => { // to add a second set of real aws credentials under a profile // named 'lisa' in ~/.aws/scality, then rename 'itSkipIfAWS' to // 'it'. - itSkipIfAWS('should return AccessDenied if user is not bucket' + - 'owner', async () => { + itSkipIfAWS('should return AccessDenied if user is not bucket' + 'owner', async () => { try { await otherAccountS3.send(new DeleteBucketCorsCommand({ Bucket: bucketName })); throw new Error('Expected AccessDenied error'); diff --git a/tests/functional/aws-node-sdk/test/bucket/deleteWebsite.js b/tests/functional/aws-node-sdk/test/bucket/deleteWebsite.js index 7518375084..aed1070f8a 100644 --- a/tests/functional/aws-node-sdk/test/bucket/deleteWebsite.js +++ b/tests/functional/aws-node-sdk/test/bucket/deleteWebsite.js @@ -1,9 +1,11 @@ const assert = require('assert'); -const { S3Client, +const { + S3Client, CreateBucketCommand, DeleteBucketCommand, DeleteBucketWebsiteCommand, - PutBucketWebsiteCommand } = require('@aws-sdk/client-s3'); + PutBucketWebsiteCommand, +} = require('@aws-sdk/client-s3'); const withV4 = require('../support/withV4'); const getConfig = require('../support/config'); @@ -40,22 +42,23 @@ describe('DELETE bucket website', () => { it('should return a 204 response', async () => { const res = await s3.send(new DeleteBucketWebsiteCommand({ Bucket: bucketName })); const statusCode = res?.$metadata?.httpStatusCode; - assert.strictEqual(statusCode, 204, - `Found unexpected statusCode ${statusCode}`); + assert.strictEqual(statusCode, 204, `Found unexpected statusCode ${statusCode}`); }); }); describe('with existing configuration', () => { beforeEach(() => { const config = new WebsiteConfigTester('index.html'); - return s3.send(new PutBucketWebsiteCommand({ - Bucket: bucketName, - WebsiteConfiguration: config - })); + return s3.send( + new PutBucketWebsiteCommand({ + Bucket: bucketName, + WebsiteConfiguration: config, + }), + ); }); - it('should delete bucket configuration successfully', () => s3.send(new - DeleteBucketWebsiteCommand({ Bucket: bucketName }))); + it('should delete bucket configuration successfully', () => + s3.send(new DeleteBucketWebsiteCommand({ Bucket: bucketName }))); it('should return AccessDenied if user is not bucket owner', async () => { try { diff --git a/tests/functional/aws-node-sdk/test/bucket/get.js b/tests/functional/aws-node-sdk/test/bucket/get.js index c872a7c659..5b0bf616b5 100644 --- a/tests/functional/aws-node-sdk/test/bucket/get.js +++ b/tests/functional/aws-node-sdk/test/bucket/get.js @@ -34,276 +34,240 @@ const vaultHost = config.vaultd?.host || 'localhost'; const tests = [ { name: 'return created objects in alphabetical order', - objectPutParams: Bucket => - [ - { Bucket, Key: 'testB/' }, - { Bucket, Key: 'testB/test.json', Body: '{}' }, - { Bucket, Key: 'testA/' }, - { Bucket, Key: 'testA/test.json', Body: '{}' }, - { Bucket, Key: 'testA/test/test.json', Body: '{}' }, - ], + objectPutParams: Bucket => [ + { Bucket, Key: 'testB/' }, + { Bucket, Key: 'testB/test.json', Body: '{}' }, + { Bucket, Key: 'testA/' }, + { Bucket, Key: 'testA/test.json', Body: '{}' }, + { Bucket, Key: 'testA/test/test.json', Body: '{}' }, + ], listObjectParams: Bucket => ({ Bucket }), assertions: (data, Bucket) => { const keys = data.Contents.map(object => object.Key); // ETag should include quotes around value - const emptyObjectHash = - '"d41d8cd98f00b204e9800998ecf8427e"'; + const emptyObjectHash = '"d41d8cd98f00b204e9800998ecf8427e"'; assert.equal(data.Name, Bucket, 'Bucket name mismatch'); - assert.deepEqual(keys, [ - 'testA/', - 'testA/test.json', - 'testA/test/test.json', - 'testB/', - 'testB/test.json', - ], 'Bucket content mismatch'); - assert.deepStrictEqual(data.Contents[0].ETag, - emptyObjectHash, 'Object hash mismatch'); + assert.deepEqual( + keys, + ['testA/', 'testA/test.json', 'testA/test/test.json', 'testB/', 'testB/test.json'], + 'Bucket content mismatch', + ); + assert.deepStrictEqual(data.Contents[0].ETag, emptyObjectHash, 'Object hash mismatch'); }, }, { name: 'return multiple common prefixes', - objectPutParams: Bucket => - [ - { Bucket, Key: 'testB/' }, - { Bucket, Key: 'testB/test.json', Body: '{}' }, - { Bucket, Key: 'testA/' }, - { Bucket, Key: 'testA/test.json', Body: '{}' }, - { Bucket, Key: 'testA/test/test.json', Body: '{}' }, - ], + objectPutParams: Bucket => [ + { Bucket, Key: 'testB/' }, + { Bucket, Key: 'testB/test.json', Body: '{}' }, + { Bucket, Key: 'testA/' }, + { Bucket, Key: 'testA/test.json', Body: '{}' }, + { Bucket, Key: 'testA/test/test.json', Body: '{}' }, + ], listObjectParams: Bucket => ({ Bucket, Delimiter: '/' }), assertions: (data, Bucket) => { const prefixes = data.CommonPrefixes.map(cp => cp.Prefix); assert.equal(data.Name, Bucket, 'Bucket name mismatch'); - assert.deepEqual(prefixes, [ - 'testA/', - 'testB/', - ], 'Bucket content mismatch'); + assert.deepEqual(prefixes, ['testA/', 'testB/'], 'Bucket content mismatch'); }, }, { name: 'list objects with percentage delimiter', - objectPutParams: Bucket => - [ - { Bucket, Key: 'testB%' }, - { Bucket, Key: 'testC%test.json', Body: '{}' }, - { Bucket, Key: 'testA%' }, - ], + objectPutParams: Bucket => [ + { Bucket, Key: 'testB%' }, + { Bucket, Key: 'testC%test.json', Body: '{}' }, + { Bucket, Key: 'testA%' }, + ], listObjectParams: Bucket => ({ Bucket, Delimiter: '%' }), assertions: data => { const prefixes = data.CommonPrefixes.map(cp => cp.Prefix); - assert.deepEqual(prefixes, [ - 'testA%', - 'testB%', - 'testC%', - ], 'Bucket content mismatch'); + assert.deepEqual(prefixes, ['testA%', 'testB%', 'testC%'], 'Bucket content mismatch'); }, }, { name: 'list object titles with white spaces', - objectPutParams: Bucket => - [ - { Bucket, Key: 'whiteSpace/' }, - { Bucket, Key: 'whiteSpace/one whiteSpace', Body: '{}' }, - { Bucket, Key: 'whiteSpace/two white spaces', Body: '{}' }, - { Bucket, Key: 'white space/' }, - { Bucket, Key: 'white space/one whiteSpace', Body: '{}' }, - { Bucket, Key: 'white space/two white spaces', Body: '{}' }, - ], + objectPutParams: Bucket => [ + { Bucket, Key: 'whiteSpace/' }, + { Bucket, Key: 'whiteSpace/one whiteSpace', Body: '{}' }, + { Bucket, Key: 'whiteSpace/two white spaces', Body: '{}' }, + { Bucket, Key: 'white space/' }, + { Bucket, Key: 'white space/one whiteSpace', Body: '{}' }, + { Bucket, Key: 'white space/two white spaces', Body: '{}' }, + ], listObjectParams: Bucket => ({ Bucket }), assertions: (data, Bucket) => { const keys = data.Contents.map(object => object.Key); assert.equal(data.Name, Bucket, 'Bucket name mismatch'); - assert.deepEqual(keys, [ - /* These object names are intentionally listed in a + assert.deepEqual( + keys, + [ + /* These object names are intentionally listed in a different order than they were created to additionally test that they are listed alphabetically. */ - 'white space/', - 'white space/one whiteSpace', - 'white space/two white spaces', - 'whiteSpace/', - 'whiteSpace/one whiteSpace', - 'whiteSpace/two white spaces', - ], 'Bucket content mismatch'); + 'white space/', + 'white space/one whiteSpace', + 'white space/two white spaces', + 'whiteSpace/', + 'whiteSpace/one whiteSpace', + 'whiteSpace/two white spaces', + ], + 'Bucket content mismatch', + ); }, }, { name: 'list object titles that contain special chars', - objectPutParams: Bucket => - [ - { Bucket, Key: 'foo&<>\'"' }, - { Bucket, Key: '*asterixObjTitle/' }, - { Bucket, Key: '*asterixObjTitle/objTitleA', Body: '{}' }, - { Bucket, Key: '*asterixObjTitle/*asterixObjTitle', - Body: '{}' }, - { Bucket, Key: '.dotObjTitle/' }, - { Bucket, Key: '.dotObjTitle/objTitleA', Body: '{}' }, - { Bucket, Key: '.dotObjTitle/.dotObjTitle', Body: '{}' }, - { Bucket, Key: '(openParenObjTitle/' }, - { Bucket, Key: '(openParenObjTitle/objTitleA', Body: '{}' }, - { Bucket, Key: '(openParenObjTitle/(openParenObjTitle', - Body: '{}' }, - { Bucket, Key: ')closeParenObjTitle/' }, - { Bucket, Key: ')closeParenObjTitle/objTitleA', Body: '{}' }, - { Bucket, Key: ')closeParenObjTitle/)closeParenObjTitle', - Body: '{}' }, - { Bucket, Key: '!exclamationPointObjTitle/' }, - { Bucket, Key: '!exclamationPointObjTitle/objTitleA', - Body: '{}' }, - { Bucket, Key: - '!exclamationPointObjTitle/!exclamationPointObjTitle', - Body: '{}' }, - { Bucket, Key: '-dashObjTitle/' }, - { Bucket, Key: '-dashObjTitle/objTitleA', Body: '{}' }, - { Bucket, Key: '-dashObjTitle/-dashObjTitle', Body: '{}' }, - { Bucket, Key: '_underscoreObjTitle/' }, - { Bucket, Key: '_underscoreObjTitle/objTitleA', Body: '{}' }, - { Bucket, Key: '_underscoreObjTitle/_underscoreObjTitle', - Body: '{}' }, - { Bucket, Key: "'apostropheObjTitle/" }, - { Bucket, Key: "'apostropheObjTitle/objTitleA", Body: '{}' }, - { Bucket, Key: "'apostropheObjTitle/'apostropheObjTitle", - Body: '{}' }, - { Bucket, Key: 'çcedilleObjTitle' }, - { Bucket, Key: 'çcedilleObjTitle/objTitleA', Body: '{}' }, - { Bucket, Key: 'çcedilleObjTitle/çcedilleObjTitle', - Body: '{}' }, - { Bucket, Key: 'дcyrillicDObjTitle' }, - { Bucket, Key: 'дcyrillicDObjTitle/objTitleA', Body: '{}' }, - { Bucket, Key: 'дcyrillicDObjTitle/дcyrillicDObjTitle', - Body: '{}' }, - { Bucket, Key: 'ñenyeObjTitle' }, - { Bucket, Key: 'ñenyeObjTitle/objTitleA', Body: '{}' }, - { Bucket, Key: 'ñenyeObjTitle/ñenyeObjTitle', Body: '{}' }, - { Bucket, Key: '山chineseMountainObjTitle' }, - { Bucket, Key: '山chineseMountainObjTitle/objTitleA', - Body: '{}' }, - { Bucket, Key: - '山chineseMountainObjTitle/山chineseMountainObjTitle', - Body: '{}' }, - { Bucket, Key: 'àaGraveLowerCaseObjTitle' }, - { Bucket, Key: 'àaGraveLowerCaseObjTitle/objTitleA', - Body: '{}' }, - { Bucket, - Key: 'àaGraveLowerCaseObjTitle/àaGraveLowerCaseObjTitle', - Body: '{}' }, - { Bucket, Key: 'ÀaGraveUpperCaseObjTitle' }, - { Bucket, Key: 'ÀaGraveUpperCaseObjTitle/objTitleA', - Body: '{}' }, - { Bucket, - Key: 'ÀaGraveUpperCaseObjTitle/ÀaGraveUpperCaseObjTitle', - Body: '{}' }, - { Bucket, Key: 'ßscharfesSObjTitle' }, - { Bucket, Key: 'ßscharfesSObjTitle/objTitleA', Body: '{}' }, - { Bucket, Key: 'ßscharfesSObjTitle/ßscharfesSObjTitle', - Body: '{}' }, - { Bucket, Key: '日japaneseMountainObjTitle' }, - { Bucket, Key: '日japaneseMountainObjTitle/objTitleA', - Body: '{}' }, - { Bucket, - Key: '日japaneseMountainObjTitle/日japaneseMountainObjTitle', - Body: '{}' }, - { Bucket, Key: 'بbaArabicObjTitle' }, - { Bucket, Key: 'بbaArabicObjTitle/objTitleA', Body: '{}' }, - { Bucket, Key: 'بbaArabicObjTitle/بbaArabicObjTitle', - Body: '{}' }, - { Bucket, - Key: 'अadevanagariHindiObjTitle' }, - { Bucket, - Key: 'अadevanagariHindiObjTitle/objTitleA', - Body: '{}' }, - { Bucket, - Key: 'अadevanagariHindiObjTitle/अadevanagariHindiObjTitle', - Body: '{}' }, - { Bucket, Key: 'éeacuteLowerCaseObjTitle' }, - { Bucket, Key: 'éeacuteLowerCaseObjTitle/objTitleA', - Body: '{}' }, - { Bucket, - Key: 'éeacuteLowerCaseObjTitle/éeacuteLowerCaseObjTitle', - Body: '{}' }, - ], + objectPutParams: Bucket => [ + { Bucket, Key: 'foo&<>\'"' }, + { Bucket, Key: '*asterixObjTitle/' }, + { Bucket, Key: '*asterixObjTitle/objTitleA', Body: '{}' }, + { Bucket, Key: '*asterixObjTitle/*asterixObjTitle', Body: '{}' }, + { Bucket, Key: '.dotObjTitle/' }, + { Bucket, Key: '.dotObjTitle/objTitleA', Body: '{}' }, + { Bucket, Key: '.dotObjTitle/.dotObjTitle', Body: '{}' }, + { Bucket, Key: '(openParenObjTitle/' }, + { Bucket, Key: '(openParenObjTitle/objTitleA', Body: '{}' }, + { Bucket, Key: '(openParenObjTitle/(openParenObjTitle', Body: '{}' }, + { Bucket, Key: ')closeParenObjTitle/' }, + { Bucket, Key: ')closeParenObjTitle/objTitleA', Body: '{}' }, + { Bucket, Key: ')closeParenObjTitle/)closeParenObjTitle', Body: '{}' }, + { Bucket, Key: '!exclamationPointObjTitle/' }, + { Bucket, Key: '!exclamationPointObjTitle/objTitleA', Body: '{}' }, + { Bucket, Key: '!exclamationPointObjTitle/!exclamationPointObjTitle', Body: '{}' }, + { Bucket, Key: '-dashObjTitle/' }, + { Bucket, Key: '-dashObjTitle/objTitleA', Body: '{}' }, + { Bucket, Key: '-dashObjTitle/-dashObjTitle', Body: '{}' }, + { Bucket, Key: '_underscoreObjTitle/' }, + { Bucket, Key: '_underscoreObjTitle/objTitleA', Body: '{}' }, + { Bucket, Key: '_underscoreObjTitle/_underscoreObjTitle', Body: '{}' }, + { Bucket, Key: "'apostropheObjTitle/" }, + { Bucket, Key: "'apostropheObjTitle/objTitleA", Body: '{}' }, + { Bucket, Key: "'apostropheObjTitle/'apostropheObjTitle", Body: '{}' }, + { Bucket, Key: 'çcedilleObjTitle' }, + { Bucket, Key: 'çcedilleObjTitle/objTitleA', Body: '{}' }, + { Bucket, Key: 'çcedilleObjTitle/çcedilleObjTitle', Body: '{}' }, + { Bucket, Key: 'дcyrillicDObjTitle' }, + { Bucket, Key: 'дcyrillicDObjTitle/objTitleA', Body: '{}' }, + { Bucket, Key: 'дcyrillicDObjTitle/дcyrillicDObjTitle', Body: '{}' }, + { Bucket, Key: 'ñenyeObjTitle' }, + { Bucket, Key: 'ñenyeObjTitle/objTitleA', Body: '{}' }, + { Bucket, Key: 'ñenyeObjTitle/ñenyeObjTitle', Body: '{}' }, + { Bucket, Key: '山chineseMountainObjTitle' }, + { Bucket, Key: '山chineseMountainObjTitle/objTitleA', Body: '{}' }, + { Bucket, Key: '山chineseMountainObjTitle/山chineseMountainObjTitle', Body: '{}' }, + { Bucket, Key: 'àaGraveLowerCaseObjTitle' }, + { Bucket, Key: 'àaGraveLowerCaseObjTitle/objTitleA', Body: '{}' }, + { Bucket, Key: 'àaGraveLowerCaseObjTitle/àaGraveLowerCaseObjTitle', Body: '{}' }, + { Bucket, Key: 'ÀaGraveUpperCaseObjTitle' }, + { Bucket, Key: 'ÀaGraveUpperCaseObjTitle/objTitleA', Body: '{}' }, + { Bucket, Key: 'ÀaGraveUpperCaseObjTitle/ÀaGraveUpperCaseObjTitle', Body: '{}' }, + { Bucket, Key: 'ßscharfesSObjTitle' }, + { Bucket, Key: 'ßscharfesSObjTitle/objTitleA', Body: '{}' }, + { Bucket, Key: 'ßscharfesSObjTitle/ßscharfesSObjTitle', Body: '{}' }, + { Bucket, Key: '日japaneseMountainObjTitle' }, + { Bucket, Key: '日japaneseMountainObjTitle/objTitleA', Body: '{}' }, + { Bucket, Key: '日japaneseMountainObjTitle/日japaneseMountainObjTitle', Body: '{}' }, + { Bucket, Key: 'بbaArabicObjTitle' }, + { Bucket, Key: 'بbaArabicObjTitle/objTitleA', Body: '{}' }, + { Bucket, Key: 'بbaArabicObjTitle/بbaArabicObjTitle', Body: '{}' }, + { Bucket, Key: 'अadevanagariHindiObjTitle' }, + { Bucket, Key: 'अadevanagariHindiObjTitle/objTitleA', Body: '{}' }, + { Bucket, Key: 'अadevanagariHindiObjTitle/अadevanagariHindiObjTitle', Body: '{}' }, + { Bucket, Key: 'éeacuteLowerCaseObjTitle' }, + { Bucket, Key: 'éeacuteLowerCaseObjTitle/objTitleA', Body: '{}' }, + { Bucket, Key: 'éeacuteLowerCaseObjTitle/éeacuteLowerCaseObjTitle', Body: '{}' }, + ], listObjectParams: Bucket => ({ Bucket }), assertions: (data, Bucket) => { const keys = data.Contents.map(object => object.Key); assert.equal(data.Name, Bucket, 'Bucket name mismatch'); - assert.deepEqual(keys, [ - /* These object names are intentionally listed in a + assert.deepEqual( + keys, + [ + /* These object names are intentionally listed in a different order than they were created to additionally test that they are listed alphabetically. */ - '!exclamationPointObjTitle/', - '!exclamationPointObjTitle/!exclamationPointObjTitle', - '!exclamationPointObjTitle/objTitleA', - "'apostropheObjTitle/", - "'apostropheObjTitle/'apostropheObjTitle", - "'apostropheObjTitle/objTitleA", - '(openParenObjTitle/', - '(openParenObjTitle/(openParenObjTitle', - '(openParenObjTitle/objTitleA', - ')closeParenObjTitle/', - ')closeParenObjTitle/)closeParenObjTitle', - ')closeParenObjTitle/objTitleA', - '*asterixObjTitle/', - '*asterixObjTitle/*asterixObjTitle', - '*asterixObjTitle/objTitleA', - '-dashObjTitle/', - '-dashObjTitle/-dashObjTitle', - '-dashObjTitle/objTitleA', - '.dotObjTitle/', - '.dotObjTitle/.dotObjTitle', - '.dotObjTitle/objTitleA', - '_underscoreObjTitle/', - '_underscoreObjTitle/_underscoreObjTitle', - '_underscoreObjTitle/objTitleA', - 'foo&<>\'"', - 'ÀaGraveUpperCaseObjTitle', - 'ÀaGraveUpperCaseObjTitle/objTitleA', - 'ÀaGraveUpperCaseObjTitle/ÀaGraveUpperCaseObjTitle', - 'ßscharfesSObjTitle', - 'ßscharfesSObjTitle/objTitleA', - 'ßscharfesSObjTitle/ßscharfesSObjTitle', - 'àaGraveLowerCaseObjTitle', - 'àaGraveLowerCaseObjTitle/objTitleA', - 'àaGraveLowerCaseObjTitle/àaGraveLowerCaseObjTitle', - 'çcedilleObjTitle', - 'çcedilleObjTitle/objTitleA', - 'çcedilleObjTitle/çcedilleObjTitle', - 'éeacuteLowerCaseObjTitle', - 'éeacuteLowerCaseObjTitle/objTitleA', - 'éeacuteLowerCaseObjTitle/éeacuteLowerCaseObjTitle', - 'ñenyeObjTitle', - 'ñenyeObjTitle/objTitleA', - 'ñenyeObjTitle/ñenyeObjTitle', - 'дcyrillicDObjTitle', - 'дcyrillicDObjTitle/objTitleA', - 'дcyrillicDObjTitle/дcyrillicDObjTitle', - 'بbaArabicObjTitle', - 'بbaArabicObjTitle/objTitleA', - 'بbaArabicObjTitle/بbaArabicObjTitle', - 'अadevanagariHindiObjTitle', - 'अadevanagariHindiObjTitle/objTitleA', - 'अadevanagariHindiObjTitle/अadevanagariHindiObjTitle', - '山chineseMountainObjTitle', - '山chineseMountainObjTitle/objTitleA', - '山chineseMountainObjTitle/山chineseMountainObjTitle', - '日japaneseMountainObjTitle', - '日japaneseMountainObjTitle/objTitleA', - '日japaneseMountainObjTitle/日japaneseMountainObjTitle', - ], 'Bucket content mismatch'); + '!exclamationPointObjTitle/', + '!exclamationPointObjTitle/!exclamationPointObjTitle', + '!exclamationPointObjTitle/objTitleA', + "'apostropheObjTitle/", + "'apostropheObjTitle/'apostropheObjTitle", + "'apostropheObjTitle/objTitleA", + '(openParenObjTitle/', + '(openParenObjTitle/(openParenObjTitle', + '(openParenObjTitle/objTitleA', + ')closeParenObjTitle/', + ')closeParenObjTitle/)closeParenObjTitle', + ')closeParenObjTitle/objTitleA', + '*asterixObjTitle/', + '*asterixObjTitle/*asterixObjTitle', + '*asterixObjTitle/objTitleA', + '-dashObjTitle/', + '-dashObjTitle/-dashObjTitle', + '-dashObjTitle/objTitleA', + '.dotObjTitle/', + '.dotObjTitle/.dotObjTitle', + '.dotObjTitle/objTitleA', + '_underscoreObjTitle/', + '_underscoreObjTitle/_underscoreObjTitle', + '_underscoreObjTitle/objTitleA', + 'foo&<>\'"', + 'ÀaGraveUpperCaseObjTitle', + 'ÀaGraveUpperCaseObjTitle/objTitleA', + 'ÀaGraveUpperCaseObjTitle/ÀaGraveUpperCaseObjTitle', + 'ßscharfesSObjTitle', + 'ßscharfesSObjTitle/objTitleA', + 'ßscharfesSObjTitle/ßscharfesSObjTitle', + 'àaGraveLowerCaseObjTitle', + 'àaGraveLowerCaseObjTitle/objTitleA', + 'àaGraveLowerCaseObjTitle/àaGraveLowerCaseObjTitle', + 'çcedilleObjTitle', + 'çcedilleObjTitle/objTitleA', + 'çcedilleObjTitle/çcedilleObjTitle', + 'éeacuteLowerCaseObjTitle', + 'éeacuteLowerCaseObjTitle/objTitleA', + 'éeacuteLowerCaseObjTitle/éeacuteLowerCaseObjTitle', + 'ñenyeObjTitle', + 'ñenyeObjTitle/objTitleA', + 'ñenyeObjTitle/ñenyeObjTitle', + 'дcyrillicDObjTitle', + 'дcyrillicDObjTitle/objTitleA', + 'дcyrillicDObjTitle/дcyrillicDObjTitle', + 'بbaArabicObjTitle', + 'بbaArabicObjTitle/objTitleA', + 'بbaArabicObjTitle/بbaArabicObjTitle', + 'अadevanagariHindiObjTitle', + 'अadevanagariHindiObjTitle/objTitleA', + 'अadevanagariHindiObjTitle/अadevanagariHindiObjTitle', + '山chineseMountainObjTitle', + '山chineseMountainObjTitle/objTitleA', + '山chineseMountainObjTitle/山chineseMountainObjTitle', + '日japaneseMountainObjTitle', + '日japaneseMountainObjTitle/objTitleA', + '日japaneseMountainObjTitle/日japaneseMountainObjTitle', + ], + 'Bucket content mismatch', + ); }, }, { name: 'list objects with special chars in CommonPrefixes', - objectPutParams: Bucket => - [ - { Bucket, Key: '&#' }, - { Bucket, Key: '"quot#' }, { Bucket, Key: '\'apos#' }, - { Bucket, Key: ' [ + { Bucket, Key: '&#' }, + { Bucket, Key: '"quot#' }, + { Bucket, Key: "'apos#" }, + { Bucket, Key: ' ({ Bucket, Delimiter: '#' }), assertions: data => { assert.deepStrictEqual(data.CommonPrefixes, [ - { Prefix: '"quot#' }, { Prefix: '&#' }, - { Prefix: '\'apos#' }, { Prefix: ' { before(done => { authenticatedBucketUtil = new BucketUtility('default', {}); unauthenticatedBucketUtil = new BucketUtility('default', {}, true); - authenticatedBucketUtil.createRandom(1) - .then(created => { - bucketName = created; - done(); - }) - .catch(done); + authenticatedBucketUtil + .createRandom(1) + .then(created => { + bucketName = created; + done(); + }) + .catch(done); }); after(done => { - authenticatedBucketUtil.deleteOne(bucketName) - .then(() => done()) - .catch(done); + authenticatedBucketUtil + .deleteOne(bucketName) + .then(() => done()) + .catch(done); }); it('should return 403 and AccessDenied on a private bucket', done => { const params = { Bucket: bucketName }; - unauthenticatedBucketUtil.s3.send(new ListObjectsCommand(params)) + unauthenticatedBucketUtil.s3 + .send(new ListObjectsCommand(params)) .then(() => { assert.fail('Expected request to fail with AccessDenied'); }) @@ -351,12 +318,13 @@ describe('GET Bucket - AWS.S3.listObjects', () => { before(done => { bucketUtil = new BucketUtility('default', sigCfg); - bucketUtil.createRandom(1) - .then(created => { - bucketName = created; - done(); - }) - .catch(done); + bucketUtil + .createRandom(1) + .then(created => { + bucketName = created; + done(); + }) + .catch(done); }); after(() => bucketUtil.deleteOne(bucketName)); @@ -373,7 +341,7 @@ describe('GET Bucket - AWS.S3.listObjects', () => { const { $metadata, ...data } = await s3.send(new ListObjectsCommand(test.listObjectParams(Bucket))); const validationSchema = { ...bucketSchema, - required: bucketSchema.required.filter(field => Object.prototype.hasOwnProperty.call(data, field)) + required: bucketSchema.required.filter(field => Object.prototype.hasOwnProperty.call(data, field)), }; const isValidResponse = tv4.validate(data, validationSchema); if (!isValidResponse) { @@ -395,7 +363,9 @@ describe('GET Bucket - AWS.S3.listObjects', () => { const { $metadata, ...data } = await s3.send(new ListObjectsV2Command(test.listObjectParams(Bucket))); const validationSchema2 = { ...bucketSchemaV2, - required: bucketSchemaV2.required.filter(field => Object.prototype.hasOwnProperty.call(data, field)) + required: bucketSchemaV2.required.filter(field => + Object.prototype.hasOwnProperty.call(data, field), + ), }; const isValidResponse = tv4.validate(data, validationSchema2); if (!isValidResponse) { @@ -406,19 +376,19 @@ describe('GET Bucket - AWS.S3.listObjects', () => { }); }); - ['&', '"quot', '\'apos', 'gt'].forEach(k => { + ['&', '"quot', "'apos", 'gt'].forEach(k => { it(`should list objects with key ${k} as Prefix`, async () => { const s3 = bucketUtil.s3; const Bucket = bucketName; const objects = [{ Bucket, Key: k }]; for (const param of objects) { - await s3.send(new PutObjectCommand(param)); - } + await s3.send(new PutObjectCommand(param)); + } const { $metadata, ...data } = await s3.send(new ListObjectsCommand({ Bucket, Prefix: k })); const validationSchema = { ...bucketSchema, - required: bucketSchema.required.filter(field => Object.prototype.hasOwnProperty.call(data, field)) + required: bucketSchema.required.filter(field => Object.prototype.hasOwnProperty.call(data, field)), }; const isValidResponse = tv4.validate(data, validationSchema); if (!isValidResponse) { @@ -429,7 +399,7 @@ describe('GET Bucket - AWS.S3.listObjects', () => { }); }); - ['&', '"quot', '\'apos', 'gt'].forEach(k => { + ['&', '"quot', "'apos", 'gt'].forEach(k => { it(`should list objects with key ${k} as Marker`, async () => { const s3 = bucketUtil.s3; const Bucket = bucketName; @@ -441,7 +411,7 @@ describe('GET Bucket - AWS.S3.listObjects', () => { const { $metadata, ...data } = await s3.send(new ListObjectsCommand({ Bucket, Marker: k })); const validationSchema = { ...bucketSchema, - required: bucketSchema.required.filter(field => Object.prototype.hasOwnProperty.call(data, field)) + required: bucketSchema.required.filter(field => Object.prototype.hasOwnProperty.call(data, field)), }; const isValidResponse = tv4.validate(data, validationSchema); if (!isValidResponse) { @@ -452,21 +422,25 @@ describe('GET Bucket - AWS.S3.listObjects', () => { }); }); - ['&', '"quot', '\'apos', 'gt'].forEach(k => { + ['&', '"quot', "'apos", 'gt'].forEach(k => { it(`should list objects with key ${k} as NextMarker`, async () => { const s3 = bucketUtil.s3; const Bucket = bucketName; - const objects = [{ Bucket, Key: k }, { Bucket, Key: 'zzz' }]; + const objects = [ + { Bucket, Key: k }, + { Bucket, Key: 'zzz' }, + ]; for (const param of objects) { await s3.send(new PutObjectCommand(param)); } - const { $metadata, ...data } = await s3.send(new ListObjectsCommand({ Bucket, MaxKeys: 1, - Delimiter: 'foo' })); + const { $metadata, ...data } = await s3.send( + new ListObjectsCommand({ Bucket, MaxKeys: 1, Delimiter: 'foo' }), + ); const validationSchema = { ...bucketSchema, - required: bucketSchema.required.filter(field => Object.prototype.hasOwnProperty.call(data, field)) + required: bucketSchema.required.filter(field => Object.prototype.hasOwnProperty.call(data, field)), }; const isValidResponse = tv4.validate(data, validationSchema); if (!isValidResponse) { @@ -477,7 +451,7 @@ describe('GET Bucket - AWS.S3.listObjects', () => { }); }); - ['&', '"quot', '\'apos', 'gt'].forEach(k => { + ['&', '"quot', "'apos", 'gt'].forEach(k => { it(`should list objects with key ${k} as StartAfter`, async () => { const s3 = bucketUtil.s3; const Bucket = bucketName; @@ -489,7 +463,9 @@ describe('GET Bucket - AWS.S3.listObjects', () => { const { $metadata, ...data } = await s3.send(new ListObjectsV2Command({ Bucket, StartAfter: k })); const validationSchema2 = { ...bucketSchemaV2, - required: bucketSchemaV2.required.filter(field => Object.prototype.hasOwnProperty.call(data, field)) + required: bucketSchemaV2.required.filter(field => + Object.prototype.hasOwnProperty.call(data, field), + ), }; const isValidResponse = tv4.validate(data, validationSchema2); if (!isValidResponse) { @@ -500,9 +476,8 @@ describe('GET Bucket - AWS.S3.listObjects', () => { }); }); - ['&', '"quot', '\'apos', 'gt'].forEach(k => { - it(`should list objects with key ${k} as ContinuationToken`, - async () => { + ['&', '"quot', "'apos", 'gt'].forEach(k => { + it(`should list objects with key ${k} as ContinuationToken`, async () => { const s3 = bucketUtil.s3; const Bucket = bucketName; const objects = [{ Bucket, Key: k }]; @@ -510,46 +485,53 @@ describe('GET Bucket - AWS.S3.listObjects', () => { for (const param of objects) { await s3.send(new PutObjectCommand(param)); } - const { $metadata, ...data } = await s3.send(new ListObjectsV2Command({ - Bucket, - ContinuationToken: generateToken(k), - })); + const { $metadata, ...data } = await s3.send( + new ListObjectsV2Command({ + Bucket, + ContinuationToken: generateToken(k), + }), + ); const validationSchema2 = { ...bucketSchemaV2, - required: bucketSchemaV2.required.filter(field => Object.prototype.hasOwnProperty.call(data, field)) + required: bucketSchemaV2.required.filter(field => + Object.prototype.hasOwnProperty.call(data, field), + ), }; const isValidResponse = tv4.validate(data, validationSchema2); if (!isValidResponse) { throw new Error(tv4.error); } - assert.deepStrictEqual( - decryptToken(data.ContinuationToken), k); + assert.deepStrictEqual(decryptToken(data.ContinuationToken), k); assert.strictEqual($metadata.httpStatusCode, 200); }); }); - ['&', '"quot', '\'apos', 'gt'].forEach(k => { - it(`should list objects with key ${k} as NextContinuationToken`, - async () => { + ['&', '"quot', "'apos", 'gt'].forEach(k => { + it(`should list objects with key ${k} as NextContinuationToken`, async () => { const s3 = bucketUtil.s3; const Bucket = bucketName; - const objects = [{ Bucket, Key: k }, { Bucket, Key: 'zzz' }]; + const objects = [ + { Bucket, Key: k }, + { Bucket, Key: 'zzz' }, + ]; for (const param of objects) { await s3.send(new PutObjectCommand(param)); } - const { $metadata, ...data } = await s3.send(new ListObjectsV2Command({ Bucket, MaxKeys: 1, - Delimiter: 'foo' })); + const { $metadata, ...data } = await s3.send( + new ListObjectsV2Command({ Bucket, MaxKeys: 1, Delimiter: 'foo' }), + ); const validationSchema2 = { ...bucketSchemaV2, - required: bucketSchemaV2.required.filter(field => Object.prototype.hasOwnProperty.call(data, field)) + required: bucketSchemaV2.required.filter(field => + Object.prototype.hasOwnProperty.call(data, field), + ), }; const isValidResponse = tv4.validate(data, validationSchema2); if (!isValidResponse) { throw new Error(tv4.error); } - assert.strictEqual( - decryptToken(data.NextContinuationToken), k); + assert.strictEqual(decryptToken(data.NextContinuationToken), k); assert.strictEqual($metadata.httpStatusCode, 200); }); }); @@ -565,31 +547,37 @@ describe('GET Bucket - AWS.S3.listObjects', () => { const iamClient = new IAMClient(iamConfig); before(async () => { - const policyRes = await iamClient.send(new CreatePolicyCommand({ - PolicyName: 'bp-bypass-policy', - PolicyDocument: JSON.stringify({ - Version: '2012-10-17', - Statement: [{ - Sid: 'AllowS3ListBucket', - Effect: 'Allow', - Action: [ - 's3:ListBucket', + const policyRes = await iamClient.send( + new CreatePolicyCommand({ + PolicyName: 'bp-bypass-policy', + PolicyDocument: JSON.stringify({ + Version: '2012-10-17', + Statement: [ + { + Sid: 'AllowS3ListBucket', + Effect: 'Allow', + Action: ['s3:ListBucket'], + Resource: ['*'], + }, ], - Resource: ['*'], - }], + }), }), - })); + ); policyWithListBucketOnly = policyRes.Policy; const userRes = await iamClient.send(new CreateUserCommand({ UserName: 'user-without-permission' })); userWithListBucketOnly = userRes.User; - await iamClient.send(new AttachUserPolicyCommand({ - UserName: userWithListBucketOnly.UserName, - PolicyArn: policyWithListBucketOnly.Arn, - })); - - const accessKeyRes = await iamClient.send(new CreateAccessKeyCommand({ - UserName: userWithListBucketOnly.UserName, - })); + await iamClient.send( + new AttachUserPolicyCommand({ + UserName: userWithListBucketOnly.UserName, + PolicyArn: policyWithListBucketOnly.Arn, + }), + ); + + const accessKeyRes = await iamClient.send( + new CreateAccessKeyCommand({ + UserName: userWithListBucketOnly.UserName, + }), + ); const accessKey = accessKeyRes.AccessKey; const s3Config = getConfig('default', { credentials: { @@ -601,10 +589,12 @@ describe('GET Bucket - AWS.S3.listObjects', () => { }); after(async () => { - await iamClient.send(new DetachUserPolicyCommand({ - UserName: userWithListBucketOnly.UserName, - PolicyArn: policyWithListBucketOnly.Arn, - })); + await iamClient.send( + new DetachUserPolicyCommand({ + UserName: userWithListBucketOnly.UserName, + PolicyArn: policyWithListBucketOnly.Arn, + }), + ); await iamClient.send(new DeletePolicyCommand({ PolicyArn: policyWithListBucketOnly.Arn })); await iamClient.send(new DeleteUserCommand({ UserName: userWithListBucketOnly.UserName })); }); @@ -613,18 +603,22 @@ describe('GET Bucket - AWS.S3.listObjects', () => { const s3 = bucketUtil.s3; const Bucket = bucketName; - await s3.send(new PutObjectCommand({ - Bucket, - Key: 'super-power-object', - Metadata: { - department: 'sales', - hr: 'true', - }, - })); - const result = await s3.send(new ListObjectsV2ExtendedCommand({ - Bucket, - ObjectAttributes: ['x-amz-meta-*', 'RestoreStatus', 'x-amz-meta-department'], - })); + await s3.send( + new PutObjectCommand({ + Bucket, + Key: 'super-power-object', + Metadata: { + department: 'sales', + hr: 'true', + }, + }), + ); + const result = await s3.send( + new ListObjectsV2ExtendedCommand({ + Bucket, + ObjectAttributes: ['x-amz-meta-*', 'RestoreStatus', 'x-amz-meta-department'], + }), + ); assert.strictEqual(result.Contents.length, 1); assert.strictEqual(result.Contents[0].Key, 'super-power-object'); @@ -636,20 +630,24 @@ describe('GET Bucket - AWS.S3.listObjects', () => { const s3 = bucketUtil.s3; const Bucket = bucketName; - await s3.send(new PutObjectCommand({ - Bucket, - Key: 'super-power-object', - Metadata: { - department: 'sales', - hr: 'true', - }, - })); + await s3.send( + new PutObjectCommand({ + Bucket, + Key: 'super-power-object', + Metadata: { + department: 'sales', + hr: 'true', + }, + }), + ); try { - await s3ClientWithListBucketOnly.send(new ListObjectsV2ExtendedCommand({ - Bucket, - ObjectAttributes: ['x-amz-meta-*', 'RestoreStatus', 'x-amz-meta-department'], - })); + await s3ClientWithListBucketOnly.send( + new ListObjectsV2ExtendedCommand({ + Bucket, + ObjectAttributes: ['x-amz-meta-*', 'RestoreStatus', 'x-amz-meta-department'], + }), + ); throw new Error('Request should have been rejected'); } catch (err) { if (err.message === 'Request should have been rejected') { @@ -664,18 +662,22 @@ describe('GET Bucket - AWS.S3.listObjects', () => { const s3 = bucketUtil.s3; const Bucket = bucketName; - await s3.send(new PutObjectCommand({ - Bucket, - Key: 'super-power-object', - Metadata: { - department: 'sales', - hr: 'true', - }, - })); - const result = await s3ClientWithListBucketOnly.send(new ListObjectsV2ExtendedCommand({ - Bucket, - ObjectAttributes: ['RestoreStatus'], - })); + await s3.send( + new PutObjectCommand({ + Bucket, + Key: 'super-power-object', + Metadata: { + department: 'sales', + hr: 'true', + }, + }), + ); + const result = await s3ClientWithListBucketOnly.send( + new ListObjectsV2ExtendedCommand({ + Bucket, + ObjectAttributes: ['RestoreStatus'], + }), + ); assert.strictEqual(result.Contents.length, 1); assert.strictEqual(result.Contents[0].Key, 'super-power-object'); @@ -698,22 +700,28 @@ describe('GET Bucket - AWS.S3.listObjects', () => { const setupIamUser = async (userName, policyDoc) => { let policy; if (policyDoc) { - const res = await iamClient.send(new CreatePolicyCommand({ - PolicyName: `${userName}-policy`, - PolicyDocument: JSON.stringify(policyDoc), - })); + const res = await iamClient.send( + new CreatePolicyCommand({ + PolicyName: `${userName}-policy`, + PolicyDocument: JSON.stringify(policyDoc), + }), + ); policy = res.Policy; } const userRes = await iamClient.send(new CreateUserCommand({ UserName: userName })); if (policy) { - await iamClient.send(new AttachUserPolicyCommand({ - UserName: userName, - PolicyArn: policy.Arn, - })); + await iamClient.send( + new AttachUserPolicyCommand({ + UserName: userName, + PolicyArn: policy.Arn, + }), + ); } - const accessKeyRes = await iamClient.send(new CreateAccessKeyCommand({ - UserName: userName, - })); + const accessKeyRes = await iamClient.send( + new CreateAccessKeyCommand({ + UserName: userName, + }), + ); const ak = accessKeyRes.AccessKey; const s3Cfg = getConfig('default', { credentials: { @@ -726,10 +734,12 @@ describe('GET Bucket - AWS.S3.listObjects', () => { const teardownIamUser = async (user, policy) => { if (policy) { - await iamClient.send(new DetachUserPolicyCommand({ - UserName: user.UserName, - PolicyArn: policy.Arn, - })); + await iamClient.send( + new DetachUserPolicyCommand({ + UserName: user.UserName, + PolicyArn: policy.Arn, + }), + ); await iamClient.send(new DeletePolicyCommand({ PolicyArn: policy.Arn })); } await iamClient.send(new DeleteUserCommand({ UserName: user.UserName })); @@ -742,18 +752,20 @@ describe('GET Bucket - AWS.S3.listObjects', () => { s3: s3ClientOptAttrsOnly, } = await setupIamUser('user-opt-attrs-only', { Version: '2012-10-17', - Statement: [{ - Sid: 'AllowOptAttrsOnly', - Effect: 'Allow', - Action: ['scality:ListBucketOptionalObjectAttributes'], - Resource: ['*'], - }], + Statement: [ + { + Sid: 'AllowOptAttrsOnly', + Effect: 'Allow', + Action: ['scality:ListBucketOptionalObjectAttributes'], + Resource: ['*'], + }, + ], })); - ({ - user: userNoPermissions, - s3: s3ClientNoPermissions, - } = await setupIamUser('user-no-permissions', null)); + ({ user: userNoPermissions, s3: s3ClientNoPermissions } = await setupIamUser( + 'user-no-permissions', + null, + )); ({ user: userListAllowAttrsDeny, @@ -786,10 +798,12 @@ describe('GET Bucket - AWS.S3.listObjects', () => { it('should reject when user has only the new permission and not s3:ListBucket', async () => { try { - await s3ClientOptAttrsOnly.send(new ListObjectsV2ExtendedCommand({ - Bucket: bucketName, - ObjectAttributes: ['x-amz-meta-foo'], - })); + await s3ClientOptAttrsOnly.send( + new ListObjectsV2ExtendedCommand({ + Bucket: bucketName, + ObjectAttributes: ['x-amz-meta-foo'], + }), + ); throw new Error('Request should have been rejected'); } catch (err) { if (err.message === 'Request should have been rejected') { @@ -802,10 +816,12 @@ describe('GET Bucket - AWS.S3.listObjects', () => { it('should reject when user has neither permission', async () => { try { - await s3ClientNoPermissions.send(new ListObjectsV2ExtendedCommand({ - Bucket: bucketName, - ObjectAttributes: ['x-amz-meta-foo'], - })); + await s3ClientNoPermissions.send( + new ListObjectsV2ExtendedCommand({ + Bucket: bucketName, + ObjectAttributes: ['x-amz-meta-foo'], + }), + ); throw new Error('Request should have been rejected'); } catch (err) { if (err.message === 'Request should have been rejected') { @@ -818,10 +834,12 @@ describe('GET Bucket - AWS.S3.listObjects', () => { it('should reject when explicit deny on the new permission overrides allow', async () => { try { - await s3ClientListAllowAttrsDeny.send(new ListObjectsV2ExtendedCommand({ - Bucket: bucketName, - ObjectAttributes: ['x-amz-meta-foo'], - })); + await s3ClientListAllowAttrsDeny.send( + new ListObjectsV2ExtendedCommand({ + Bucket: bucketName, + ObjectAttributes: ['x-amz-meta-foo'], + }), + ); throw new Error('Request should have been rejected'); } catch (err) { if (err.message === 'Request should have been rejected') { @@ -840,34 +858,40 @@ describe('GET Bucket - AWS.S3.listObjects', () => { before(async () => { const userName = 'user-with-both-perms'; - const policyRes = await iamClient.send(new CreatePolicyCommand({ - PolicyName: `${userName}-policy`, - PolicyDocument: JSON.stringify({ - Version: '2012-10-17', - Statement: [ - { - Effect: 'Allow', - Action: ['s3:ListBucket'], - Resource: ['*'], - }, - { - Effect: 'Allow', - Action: ['scality:ListBucketOptionalObjectAttributes'], - Resource: ['*'], - }, - ], + const policyRes = await iamClient.send( + new CreatePolicyCommand({ + PolicyName: `${userName}-policy`, + PolicyDocument: JSON.stringify({ + Version: '2012-10-17', + Statement: [ + { + Effect: 'Allow', + Action: ['s3:ListBucket'], + Resource: ['*'], + }, + { + Effect: 'Allow', + Action: ['scality:ListBucketOptionalObjectAttributes'], + Resource: ['*'], + }, + ], + }), }), - })); + ); policyWithBothPerms = policyRes.Policy; const userRes = await iamClient.send(new CreateUserCommand({ UserName: userName })); userWithBothPerms = userRes.User; - await iamClient.send(new AttachUserPolicyCommand({ - UserName: userName, - PolicyArn: policyWithBothPerms.Arn, - })); - const accessKeyRes = await iamClient.send(new CreateAccessKeyCommand({ - UserName: userName, - })); + await iamClient.send( + new AttachUserPolicyCommand({ + UserName: userName, + PolicyArn: policyWithBothPerms.Arn, + }), + ); + const accessKeyRes = await iamClient.send( + new CreateAccessKeyCommand({ + UserName: userName, + }), + ); const ak = accessKeyRes.AccessKey; const s3Cfg = getConfig('default', { credentials: { @@ -879,47 +903,52 @@ describe('GET Bucket - AWS.S3.listObjects', () => { }); after(async () => { - await iamClient.send(new DetachUserPolicyCommand({ - UserName: userWithBothPerms.UserName, - PolicyArn: policyWithBothPerms.Arn, - })); + await iamClient.send( + new DetachUserPolicyCommand({ + UserName: userWithBothPerms.UserName, + PolicyArn: policyWithBothPerms.Arn, + }), + ); await iamClient.send(new DeletePolicyCommand({ PolicyArn: policyWithBothPerms.Arn })); await iamClient.send(new DeleteUserCommand({ UserName: userWithBothPerms.UserName })); }); afterEach(async () => { - await bucketUtil.s3 - .send(new DeleteBucketPolicyCommand({ Bucket: bucketName })) - .catch(() => {}); + await bucketUtil.s3.send(new DeleteBucketPolicyCommand({ Bucket: bucketName })).catch(() => {}); }); it('should allow when the bucket policy supplies scality:ListBucketOptionalObjectAttributes that IAM lacks', async () => { - await bucketUtil.s3.send(new PutBucketPolicyCommand({ - Bucket: bucketName, - Policy: JSON.stringify({ - Version: '2012-10-17', - Statement: [{ - Effect: 'Allow', - Principal: { AWS: userWithListBucketOnly.Arn }, - Action: ['scality:ListBucketOptionalObjectAttributes'], - Resource: [ - `arn:aws:s3:::${bucketName}`, - `arn:aws:s3:::${bucketName}/*`, + await bucketUtil.s3.send( + new PutBucketPolicyCommand({ + Bucket: bucketName, + Policy: JSON.stringify({ + Version: '2012-10-17', + Statement: [ + { + Effect: 'Allow', + Principal: { AWS: userWithListBucketOnly.Arn }, + Action: ['scality:ListBucketOptionalObjectAttributes'], + Resource: [`arn:aws:s3:::${bucketName}`, `arn:aws:s3:::${bucketName}/*`], + }, ], - }], + }), }), - })); + ); - await bucketUtil.s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: 'object-with-color', - Metadata: { color: 'red' }, - })); + await bucketUtil.s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: 'object-with-color', + Metadata: { color: 'red' }, + }), + ); - const result = await s3ClientWithListBucketOnly.send(new ListObjectsV2ExtendedCommand({ - Bucket: bucketName, - ObjectAttributes: ['x-amz-meta-color'], - })); + const result = await s3ClientWithListBucketOnly.send( + new ListObjectsV2ExtendedCommand({ + Bucket: bucketName, + ObjectAttributes: ['x-amz-meta-color'], + }), + ); assert.ok(Array.isArray(result.Contents)); assert.strictEqual(result.Contents.length, 1); @@ -927,27 +956,30 @@ describe('GET Bucket - AWS.S3.listObjects', () => { }); it('should reject when the bucket policy denies the new action even if IAM allows it', async () => { - await bucketUtil.s3.send(new PutBucketPolicyCommand({ - Bucket: bucketName, - Policy: JSON.stringify({ - Version: '2012-10-17', - Statement: [{ - Effect: 'Deny', - Principal: { AWS: userWithBothPerms.Arn }, - Action: ['scality:ListBucketOptionalObjectAttributes'], - Resource: [ - `arn:aws:s3:::${bucketName}`, - `arn:aws:s3:::${bucketName}/*`, + await bucketUtil.s3.send( + new PutBucketPolicyCommand({ + Bucket: bucketName, + Policy: JSON.stringify({ + Version: '2012-10-17', + Statement: [ + { + Effect: 'Deny', + Principal: { AWS: userWithBothPerms.Arn }, + Action: ['scality:ListBucketOptionalObjectAttributes'], + Resource: [`arn:aws:s3:::${bucketName}`, `arn:aws:s3:::${bucketName}/*`], + }, ], - }], + }), }), - })); + ); try { - await s3ClientWithBothPerms.send(new ListObjectsV2ExtendedCommand({ - Bucket: bucketName, - ObjectAttributes: ['x-amz-meta-foo'], - })); + await s3ClientWithBothPerms.send( + new ListObjectsV2ExtendedCommand({ + Bucket: bucketName, + ObjectAttributes: ['x-amz-meta-foo'], + }), + ); throw new Error('Request should have been rejected'); } catch (err) { if (err.message === 'Request should have been rejected') { diff --git a/tests/functional/aws-node-sdk/test/bucket/getBucketEncryption.js b/tests/functional/aws-node-sdk/test/bucket/getBucketEncryption.js index 8327b1970d..c40ac495fe 100644 --- a/tests/functional/aws-node-sdk/test/bucket/getBucketEncryption.js +++ b/tests/functional/aws-node-sdk/test/bucket/getBucketEncryption.js @@ -1,8 +1,10 @@ const assert = require('assert'); -const { S3Client, +const { + S3Client, CreateBucketCommand, DeleteBucketCommand, - GetBucketEncryptionCommand } = require('@aws-sdk/client-s3'); + GetBucketEncryptionCommand, +} = require('@aws-sdk/client-s3'); const checkError = require('../../lib/utility/checkError'); const getConfig = require('../support/config'); @@ -38,9 +40,9 @@ describe('aws-sdk test get bucket encryption', () => { const config = getConfig('default', { signatureVersion: 'v4' }); s3 = new S3Client(config); await new Promise((resolve, reject) => { - metadata.setup(err => err ? reject(err) : resolve()); + metadata.setup(err => (err ? reject(err) : resolve())); }); - }); + }); beforeEach(() => s3.send(new CreateBucketCommand({ Bucket: bucketName }))); @@ -75,8 +77,13 @@ describe('aws-sdk test get bucket encryption', () => { }); it('should include KMSMasterKeyID if user has configured a custom master key', async () => { - await setEncryptionInfo({ cryptoScheme: 1, algorithm: 'aws:kms', masterKeyId: '12345', - configuredMasterKeyId: '54321', mandatory: true }); + await setEncryptionInfo({ + cryptoScheme: 1, + algorithm: 'aws:kms', + masterKeyId: '12345', + configuredMasterKeyId: '54321', + mandatory: true, + }); const { $metadata, ...res } = await s3.send(new GetBucketEncryptionCommand({ Bucket: bucketName })); assert.deepStrictEqual(res, { ServerSideEncryptionConfiguration: { diff --git a/tests/functional/aws-node-sdk/test/bucket/getBucketLifecycle.js b/tests/functional/aws-node-sdk/test/bucket/getBucketLifecycle.js index b0f7aaf270..1552b9850f 100644 --- a/tests/functional/aws-node-sdk/test/bucket/getBucketLifecycle.js +++ b/tests/functional/aws-node-sdk/test/bucket/getBucketLifecycle.js @@ -1,10 +1,12 @@ const assert = require('assert'); const { errors } = require('arsenal'); -const { S3Client, +const { + S3Client, CreateBucketCommand, DeleteBucketCommand, GetBucketLifecycleConfigurationCommand, - PutBucketLifecycleConfigurationCommand } = require('@aws-sdk/client-s3'); + PutBucketLifecycleConfigurationCommand, +} = require('@aws-sdk/client-s3'); const getConfig = require('../support/config'); const BucketUtility = require('../../lib/utility/bucket-util'); @@ -15,11 +17,16 @@ function assertError(err, expectedErr) { if (expectedErr === null) { assert.strictEqual(err, null, `expected no error but got '${err}'`); } else { - assert.strictEqual(err.name, expectedErr, 'incorrect error response ' + - `code: should be '${expectedErr}' but got '${err.name}'`); - assert.strictEqual(err.$metadata.httpStatusCode, errors[expectedErr].code, - 'incorrect error status code: should be 400 but got ' + - `'${err.$metadata.httpStatusCode}'`); + assert.strictEqual( + err.name, + expectedErr, + 'incorrect error response ' + `code: should be '${expectedErr}' but got '${err.name}'`, + ); + assert.strictEqual( + err.$metadata.httpStatusCode, + errors[expectedErr].code, + 'incorrect error status code: should be 400 but got ' + `'${err.$metadata.httpStatusCode}'`, + ); } } @@ -57,8 +64,7 @@ describe('aws-sdk test get bucket lifecycle', () => { } }); - it('should return NoSuchLifecycleConfiguration error if no lifecycle ' + - 'put to bucket', async () => { + it('should return NoSuchLifecycleConfiguration error if no lifecycle ' + 'put to bucket', async () => { try { await s3.send(new GetBucketLifecycleConfigurationCommand({ Bucket: bucket })); throw new Error('Expected NoSuchLifecycleConfiguration error'); @@ -68,17 +74,21 @@ describe('aws-sdk test get bucket lifecycle', () => { }); it('should get bucket lifecycle config with top-level prefix', async () => { - await s3.send(new PutBucketLifecycleConfigurationCommand({ - Bucket: bucket, - LifecycleConfiguration: { - Rules: [{ - ID: 'test-id', - Status: 'Enabled', - Prefix: '', - Expiration: { Days: 1 }, - }], - }, - })); + await s3.send( + new PutBucketLifecycleConfigurationCommand({ + Bucket: bucket, + LifecycleConfiguration: { + Rules: [ + { + ID: 'test-id', + Status: 'Enabled', + Prefix: '', + Expiration: { Days: 1 }, + }, + ], + }, + }), + ); const res = await s3.send(new GetBucketLifecycleConfigurationCommand({ Bucket: bucket })); assert.strictEqual(res.Rules.length, 1); assert.deepStrictEqual(res.Rules[0], { @@ -90,17 +100,21 @@ describe('aws-sdk test get bucket lifecycle', () => { }); it('should get bucket lifecycle config with filter prefix', async () => { - await s3.send(new PutBucketLifecycleConfigurationCommand({ - Bucket: bucket, - LifecycleConfiguration: { - Rules: [{ - ID: 'test-id', - Status: 'Enabled', - Filter: { Prefix: '' }, - Expiration: { Days: 1 }, - }], - }, - })); + await s3.send( + new PutBucketLifecycleConfigurationCommand({ + Bucket: bucket, + LifecycleConfiguration: { + Rules: [ + { + ID: 'test-id', + Status: 'Enabled', + Filter: { Prefix: '' }, + Expiration: { Days: 1 }, + }, + ], + }, + }), + ); const res = await s3.send(new GetBucketLifecycleConfigurationCommand({ Bucket: bucket })); assert.strictEqual(res.Rules.length, 1); assert.deepStrictEqual(res.Rules[0], { @@ -112,27 +126,31 @@ describe('aws-sdk test get bucket lifecycle', () => { }); it('should get bucket lifecycle config with filter prefix and tags', async () => { - await s3.send(new PutBucketLifecycleConfigurationCommand({ - Bucket: bucket, - LifecycleConfiguration: { - Rules: [{ - ID: 'test-id', - Status: 'Enabled', - Filter: { - And: { - Prefix: '', - Tags: [ - { - Key: 'key', - Value: 'value', + await s3.send( + new PutBucketLifecycleConfigurationCommand({ + Bucket: bucket, + LifecycleConfiguration: { + Rules: [ + { + ID: 'test-id', + Status: 'Enabled', + Filter: { + And: { + Prefix: '', + Tags: [ + { + Key: 'key', + Value: 'value', + }, + ], }, - ], + }, + Expiration: { Days: 1 }, }, - }, - Expiration: { Days: 1 }, - }], - }, - })); + ], + }, + }), + ); const res = await s3.send(new GetBucketLifecycleConfigurationCommand({ Bucket: bucket })); assert.strictEqual(res.Rules.length, 1); assert.deepStrictEqual(res.Rules[0], { diff --git a/tests/functional/aws-node-sdk/test/bucket/getBucketLogging.js b/tests/functional/aws-node-sdk/test/bucket/getBucketLogging.js index dcdb101d75..1cd628c156 100644 --- a/tests/functional/aws-node-sdk/test/bucket/getBucketLogging.js +++ b/tests/functional/aws-node-sdk/test/bucket/getBucketLogging.js @@ -1,9 +1,5 @@ const assert = require('assert'); -const { - CreateBucketCommand, - GetBucketLoggingCommand, - PutBucketLoggingCommand, -} = require('@aws-sdk/client-s3'); +const { CreateBucketCommand, GetBucketLoggingCommand, PutBucketLoggingCommand } = require('@aws-sdk/client-s3'); const withV4 = require('../support/withV4'); const BucketUtility = require('../../lib/utility/bucket-util'); @@ -30,7 +26,9 @@ function cleanUp(bucketUtil, cb) { throw err; } }), - ]).then(() => cb()).catch(err => cb(err)); + ]) + .then(() => cb()) + .catch(err => cb(err)); } describe('GET bucket logging', () => { @@ -38,10 +36,14 @@ describe('GET bucket logging', () => { const bucketUtil = new BucketUtility('default', sigCfg); const s3 = bucketUtil.s3; - after(done => { cleanUp(bucketUtil, done); }); + after(done => { + cleanUp(bucketUtil, done); + }); describe('without existing bucket', () => { - afterEach(done => { cleanUp(bucketUtil, done); }); + afterEach(done => { + cleanUp(bucketUtil, done); + }); it('should return NoSuchBucket', done => { s3.send(new GetBucketLoggingCommand({ Bucket: bucketName })) @@ -58,7 +60,9 @@ describe('GET bucket logging', () => { }); describe('on bucket without logging configuration', () => { - afterEach(done => { cleanUp(bucketUtil, done); }); + afterEach(done => { + cleanUp(bucketUtil, done); + }); beforeEach(done => { process.stdout.write('Creating bucket without logging\n'); @@ -86,16 +90,22 @@ describe('GET bucket logging', () => { }); describe('with existing logging configuration', () => { - afterEach(done => { cleanUp(bucketUtil, done); }); + afterEach(done => { + cleanUp(bucketUtil, done); + }); beforeEach(done => { process.stdout.write('Creating buckets and setting logging\n'); s3.send(new CreateBucketCommand({ Bucket: bucketName })) .then(() => s3.send(new CreateBucketCommand({ Bucket: targetBucket }))) - .then(() => s3.send(new PutBucketLoggingCommand({ - Bucket: bucketName, - BucketLoggingStatus: validLoggingConfig, - }))) + .then(() => + s3.send( + new PutBucketLoggingCommand({ + Bucket: bucketName, + BucketLoggingStatus: validLoggingConfig, + }), + ), + ) .then(() => done()) .catch(done); }); diff --git a/tests/functional/aws-node-sdk/test/bucket/getBucketNotification.js b/tests/functional/aws-node-sdk/test/bucket/getBucketNotification.js index 9ca98ef370..2f9a953d94 100644 --- a/tests/functional/aws-node-sdk/test/bucket/getBucketNotification.js +++ b/tests/functional/aws-node-sdk/test/bucket/getBucketNotification.js @@ -1,21 +1,25 @@ const assert = require('assert'); const { errors } = require('arsenal'); -const { S3Client, +const { + S3Client, CreateBucketCommand, DeleteBucketCommand, GetBucketNotificationConfigurationCommand, - PutBucketNotificationConfigurationCommand } = require('@aws-sdk/client-s3'); + PutBucketNotificationConfigurationCommand, +} = require('@aws-sdk/client-s3'); const getConfig = require('../support/config'); const BucketUtility = require('../../lib/utility/bucket-util'); const bucket = 'notificationtestbucket'; const notificationConfig = { - QueueConfigurations: [{ - Events: ['s3:ObjectCreated:*'], - QueueArn: 'arn:scality:bucketnotif:::target1', - Id: 'test-id', - }], + QueueConfigurations: [ + { + Events: ['s3:ObjectCreated:*'], + QueueArn: 'arn:scality:bucketnotif:::target1', + Id: 'test-id', + }, + ], }; // Check for the expected error response code and status code. @@ -61,14 +65,17 @@ describe('aws-sdk test get bucket notification', () => { } }); - it('should not return an error if no notification configuration ' + - 'put to bucket', () => s3.send(new GetBucketNotificationConfigurationCommand({ Bucket: bucket }))); + it('should not return an error if no notification configuration ' + 'put to bucket', () => + s3.send(new GetBucketNotificationConfigurationCommand({ Bucket: bucket })), + ); it('should get bucket notification config', async () => { - await s3.send(new PutBucketNotificationConfigurationCommand({ - Bucket: bucket, - NotificationConfiguration: notificationConfig, - })); + await s3.send( + new PutBucketNotificationConfigurationCommand({ + Bucket: bucket, + NotificationConfiguration: notificationConfig, + }), + ); const res = await s3.send(new GetBucketNotificationConfigurationCommand({ Bucket: bucket })); assert.deepStrictEqual(res.QueueConfigurations, notificationConfig.QueueConfigurations); }); diff --git a/tests/functional/aws-node-sdk/test/bucket/getBucketObjectLock.js b/tests/functional/aws-node-sdk/test/bucket/getBucketObjectLock.js index 11eaca40f6..9615fa8cc5 100644 --- a/tests/functional/aws-node-sdk/test/bucket/getBucketObjectLock.js +++ b/tests/functional/aws-node-sdk/test/bucket/getBucketObjectLock.js @@ -1,9 +1,11 @@ const assert = require('assert'); -const { S3Client, +const { + S3Client, CreateBucketCommand, DeleteBucketCommand, GetObjectLockConfigurationCommand, - PutObjectLockConfigurationCommand } = require('@aws-sdk/client-s3'); + PutObjectLockConfigurationCommand, +} = require('@aws-sdk/client-s3'); const checkError = require('../../lib/utility/checkError'); const getConfig = require('../support/config'); @@ -61,10 +63,14 @@ describe('aws-sdk test get bucket object lock', () => { }); describe('config rules', () => { - beforeEach(() => s3.send(new CreateBucketCommand({ - Bucket: bucket, - ObjectLockEnabledForBucket: true, - }))); + beforeEach(() => + s3.send( + new CreateBucketCommand({ + Bucket: bucket, + ObjectLockEnabledForBucket: true, + }), + ), + ); afterEach(() => s3.send(new DeleteBucketCommand({ Bucket: bucket }))); @@ -78,10 +84,12 @@ describe('aws-sdk test get bucket object lock', () => { }); it('should get bucket object lock config', async () => { - await s3.send(new PutObjectLockConfigurationCommand({ - Bucket: bucket, - ObjectLockConfiguration: objectLockConfig, - })); + await s3.send( + new PutObjectLockConfigurationCommand({ + Bucket: bucket, + ObjectLockConfiguration: objectLockConfig, + }), + ); const res = await s3.send(new GetObjectLockConfigurationCommand({ Bucket: bucket })); assert.deepStrictEqual(res.ObjectLockConfiguration, objectLockConfig); }); diff --git a/tests/functional/aws-node-sdk/test/bucket/getBucketPolicy.js b/tests/functional/aws-node-sdk/test/bucket/getBucketPolicy.js index 25b8d66eb2..1fd515b16f 100644 --- a/tests/functional/aws-node-sdk/test/bucket/getBucketPolicy.js +++ b/tests/functional/aws-node-sdk/test/bucket/getBucketPolicy.js @@ -1,10 +1,12 @@ const assert = require('assert'); const { errors } = require('arsenal'); -const { S3Client, +const { + S3Client, CreateBucketCommand, DeleteBucketCommand, GetBucketPolicyCommand, - PutBucketPolicyCommand } = require('@aws-sdk/client-s3'); + PutBucketPolicyCommand, +} = require('@aws-sdk/client-s3'); const getConfig = require('../support/config'); const BucketUtility = require('../../lib/utility/bucket-util'); @@ -12,13 +14,15 @@ const BucketUtility = require('../../lib/utility/bucket-util'); const bucket = 'getbucketpolicy-testbucket'; const bucketPolicy = { Version: '2012-10-17', - Statement: [{ - Sid: 'testid', - Effect: 'Allow', - Principal: '*', - Action: 's3:putBucketPolicy', - Resource: `arn:aws:s3:::${bucket}`, - }], + Statement: [ + { + Sid: 'testid', + Effect: 'Allow', + Principal: '*', + Action: 's3:putBucketPolicy', + Resource: `arn:aws:s3:::${bucket}`, + }, + ], }; const expectedPolicy = { Sid: 'testid', @@ -32,11 +36,16 @@ function assertError(err, expectedErr) { if (expectedErr === null) { assert.strictEqual(err, null, `expected no error but got '${err}'`); } else { - assert.strictEqual(err.name, expectedErr, 'incorrect error response ' + - `code: should be '${expectedErr}' but got '${err.name}'`); - assert.strictEqual(err.$metadata.httpStatusCode, errors[expectedErr].code, - 'incorrect error status code: should be 400 but got ' + - `'${err.$metadata.httpStatusCode}'`); + assert.strictEqual( + err.name, + expectedErr, + 'incorrect error response ' + `code: should be '${expectedErr}' but got '${err.name}'`, + ); + assert.strictEqual( + err.$metadata.httpStatusCode, + errors[expectedErr].code, + 'incorrect error status code: should be 400 but got ' + `'${err.$metadata.httpStatusCode}'`, + ); } } @@ -78,10 +87,12 @@ describe('aws-sdk test get bucket policy', () => { }); it('should get bucket policy', async () => { - await s3.send(new PutBucketPolicyCommand({ - Bucket: bucket, - Policy: JSON.stringify(bucketPolicy), - })); + await s3.send( + new PutBucketPolicyCommand({ + Bucket: bucket, + Policy: JSON.stringify(bucketPolicy), + }), + ); const res = await s3.send(new GetBucketPolicyCommand({ Bucket: bucket })); const parsedRes = JSON.parse(res.Policy); assert.deepStrictEqual(parsedRes.Statement[0], expectedPolicy); diff --git a/tests/functional/aws-node-sdk/test/bucket/getBucketQuota.js b/tests/functional/aws-node-sdk/test/bucket/getBucketQuota.js index 35fd316623..815c606f0b 100644 --- a/tests/functional/aws-node-sdk/test/bucket/getBucketQuota.js +++ b/tests/functional/aws-node-sdk/test/bucket/getBucketQuota.js @@ -1,6 +1,4 @@ -const { S3Client, - CreateBucketCommand, - DeleteBucketCommand } = require('@aws-sdk/client-s3'); +const { S3Client, CreateBucketCommand, DeleteBucketCommand } = require('@aws-sdk/client-s3'); const assert = require('assert'); const getConfig = require('../support/config'); const sendRequest = require('../quota/tooling').sendRequest; @@ -49,7 +47,7 @@ describe('Test get bucket quota', () => { await sendRequest('GET', '127.0.0.1:8000', `/${bucket}/?quota=true`); assert.fail('Expected NoSuchQuota error'); } catch (err) { - assert.strictEqual(err.Error.Code[0], 'NoSuchQuota'); + assert.strictEqual(err.Error.Code[0], 'NoSuchQuota'); } }); diff --git a/tests/functional/aws-node-sdk/test/bucket/getBucketRateLimit.js b/tests/functional/aws-node-sdk/test/bucket/getBucketRateLimit.js index 178539ed02..f2e840aaa0 100644 --- a/tests/functional/aws-node-sdk/test/bucket/getBucketRateLimit.js +++ b/tests/functional/aws-node-sdk/test/bucket/getBucketRateLimit.js @@ -1,9 +1,5 @@ const assert = require('assert'); -const { - S3Client, - CreateBucketCommand, - DeleteBucketCommand, -} = require('@aws-sdk/client-s3'); +const { S3Client, CreateBucketCommand, DeleteBucketCommand } = require('@aws-sdk/client-s3'); const getConfig = require('../support/config'); const { sendRateLimitRequest, skipIfRateLimitDisabled } = require('../rateLimit/tooling'); @@ -35,12 +31,15 @@ skipIfRateLimitDisabled('Test get bucket rate limit', () => { it('should return the rate limit config', async () => { try { // First set the rate limit config - await sendRateLimitRequest('PUT', '127.0.0.1:8000', - `/${bucket}/?rate-limit`, JSON.stringify(rateLimitConfig)); + await sendRateLimitRequest( + 'PUT', + '127.0.0.1:8000', + `/${bucket}/?rate-limit`, + JSON.stringify(rateLimitConfig), + ); // Then get it - const data = await sendRateLimitRequest('GET', '127.0.0.1:8000', - `/${bucket}/?rate-limit`); + const data = await sendRateLimitRequest('GET', '127.0.0.1:8000', `/${bucket}/?rate-limit`); assert.strictEqual(data.RequestsPerSecond.Limit, 100); } catch (err) { assert.ifError(err); diff --git a/tests/functional/aws-node-sdk/test/bucket/getBucketReplication.js b/tests/functional/aws-node-sdk/test/bucket/getBucketReplication.js index 149d4c8511..f1e3b9412e 100644 --- a/tests/functional/aws-node-sdk/test/bucket/getBucketReplication.js +++ b/tests/functional/aws-node-sdk/test/bucket/getBucketReplication.js @@ -1,10 +1,12 @@ const assert = require('assert'); -const { S3Client, +const { + S3Client, CreateBucketCommand, DeleteBucketCommand, GetBucketReplicationCommand, PutBucketReplicationCommand, - PutBucketVersioningCommand } = require('@aws-sdk/client-s3'); + PutBucketVersioningCommand, +} = require('@aws-sdk/client-s3'); const { errorInstances } = require('arsenal'); const getConfig = require('../support/config'); @@ -13,8 +15,7 @@ const BucketUtility = require('../../lib/utility/bucket-util'); const bucket = 'source-bucket'; const replicationConfig = { - Role: 'arn:aws:iam::account-id:role/src-resource,' + - 'arn:aws:iam::account-id:role/dest-resource', + Role: 'arn:aws:iam::account-id:role/src-resource,' + 'arn:aws:iam::account-id:role/dest-resource', Rules: [ { Destination: { Bucket: 'arn:aws:s3:::destination-bucket' }, @@ -32,33 +33,40 @@ describe('aws-node-sdk test getBucketReplication', () => { beforeEach(async () => { const config = getConfig('default', { signatureVersion: 'v4' }); s3 = new S3Client(config); - otherAccountS3 = new BucketUtility('lisa', {}).s3; + otherAccountS3 = new BucketUtility('lisa', {}).s3; await s3.send(new CreateBucketCommand({ Bucket: bucket })); - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { - Status: 'Enabled', - }, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { + Status: 'Enabled', + }, + }), + ); }); afterEach(() => s3.send(new DeleteBucketCommand({ Bucket: bucket }))); - it("should return 'ReplicationConfigurationNotFoundError' if bucket does " + - 'not have a replication configuration', async () => { - try { - await s3.send(new GetBucketReplicationCommand({ Bucket: bucket })); - throw new Error('Expected ReplicationConfigurationNotFoundError'); - } catch (err) { - assert(errorInstances.ReplicationConfigurationNotFoundError.is[err.Code]); - } - }); + it( + "should return 'ReplicationConfigurationNotFoundError' if bucket does " + + 'not have a replication configuration', + async () => { + try { + await s3.send(new GetBucketReplicationCommand({ Bucket: bucket })); + throw new Error('Expected ReplicationConfigurationNotFoundError'); + } catch (err) { + assert(errorInstances.ReplicationConfigurationNotFoundError.is[err.Code]); + } + }, + ); it('should get the replication configuration that was put on a bucket', async () => { - await s3.send(new PutBucketReplicationCommand({ - Bucket: bucket, - ReplicationConfiguration: replicationConfig, - })); + await s3.send( + new PutBucketReplicationCommand({ + Bucket: bucket, + ReplicationConfiguration: replicationConfig, + }), + ); const data = await s3.send(new GetBucketReplicationCommand({ Bucket: bucket })); const expectedObj = { ReplicationConfiguration: replicationConfig, diff --git a/tests/functional/aws-node-sdk/test/bucket/getBucketTagging.js b/tests/functional/aws-node-sdk/test/bucket/getBucketTagging.js index 1b0605709d..2a2e1f24eb 100644 --- a/tests/functional/aws-node-sdk/test/bucket/getBucketTagging.js +++ b/tests/functional/aws-node-sdk/test/bucket/getBucketTagging.js @@ -1,9 +1,11 @@ const assertError = require('../../../../utilities/bucketTagging-util'); -const { S3Client, +const { + S3Client, CreateBucketCommand, DeleteBucketCommand, GetBucketTaggingCommand, - PutBucketTaggingCommand } = require('@aws-sdk/client-s3'); + PutBucketTaggingCommand, +} = require('@aws-sdk/client-s3'); const assert = require('assert'); const getConfig = require('../support/config'); @@ -19,16 +21,18 @@ describe('aws-sdk test get bucket tagging', () => { }); beforeEach(() => s3.send(new CreateBucketCommand({ Bucket: bucket }))); - + afterEach(() => s3.send(new DeleteBucketCommand({ Bucket: bucket }))); it('should return accessDenied if expected bucket owner does not match', async () => { try { - await s3.send(new GetBucketTaggingCommand({ - AccountId: s3.AccountId, - Bucket: bucket, - ExpectedBucketOwner: '944690102203', - })); + await s3.send( + new GetBucketTaggingCommand({ + AccountId: s3.AccountId, + Bucket: bucket, + ExpectedBucketOwner: '944690102203', + }), + ); throw new Error('Expected AccessDenied error'); } catch (err) { assertError(err, 'AccessDenied'); @@ -37,11 +41,13 @@ describe('aws-sdk test get bucket tagging', () => { it('should not return accessDenied if expected bucket owner matches', async () => { try { - await s3.send(new GetBucketTaggingCommand({ - AccountId: s3.AccountId, - Bucket: bucket, - ExpectedBucketOwner: s3.AccountId - })); + await s3.send( + new GetBucketTaggingCommand({ + AccountId: s3.AccountId, + Bucket: bucket, + ExpectedBucketOwner: s3.AccountId, + }), + ); throw new Error('Expected NoSuchTagSet error'); } catch (err) { assertError(err, 'NoSuchTagSet'); @@ -57,17 +63,21 @@ describe('aws-sdk test get bucket tagging', () => { }, ], }; - await s3.send(new PutBucketTaggingCommand({ - AccountId: s3.AccountId, - Tagging: tagSet, - Bucket: bucket, - ExpectedBucketOwner: s3.AccountId - })); - const result = await s3.send(new GetBucketTaggingCommand({ - AccountId: s3.AccountId, - Bucket: bucket, - ExpectedBucketOwner: s3.AccountId - })); + await s3.send( + new PutBucketTaggingCommand({ + AccountId: s3.AccountId, + Tagging: tagSet, + Bucket: bucket, + ExpectedBucketOwner: s3.AccountId, + }), + ); + const result = await s3.send( + new GetBucketTaggingCommand({ + AccountId: s3.AccountId, + Bucket: bucket, + ExpectedBucketOwner: s3.AccountId, + }), + ); assert.deepStrictEqual(result.TagSet, tagSet.TagSet); }); diff --git a/tests/functional/aws-node-sdk/test/bucket/getCors.js b/tests/functional/aws-node-sdk/test/bucket/getCors.js index ba8a1636bc..2573cca92f 100644 --- a/tests/functional/aws-node-sdk/test/bucket/getCors.js +++ b/tests/functional/aws-node-sdk/test/bucket/getCors.js @@ -1,9 +1,11 @@ const assert = require('assert'); -const { S3Client, +const { + S3Client, CreateBucketCommand, DeleteBucketCommand, GetBucketCorsCommand, - PutBucketCorsCommand } = require('@aws-sdk/client-s3'); + PutBucketCorsCommand, +} = require('@aws-sdk/client-s3'); const withV4 = require('../support/withV4'); const getConfig = require('../support/config'); @@ -18,24 +20,27 @@ describe('GET bucket cors', () => { afterEach(() => s3.send(new DeleteBucketCommand({ Bucket: bucketName }))); describe('on bucket with existing cors configuration', () => { - const sampleCors = { CORSRules: [ - { AllowedMethods: ['PUT', 'POST', 'DELETE'], - AllowedOrigins: ['http://www.example.com'], - AllowedHeaders: ['*'], - MaxAgeSeconds: 3000, - ExposeHeaders: ['x-amz-server-side-encryption'] }, - { AllowedMethods: ['GET'], - AllowedOrigins: ['*'], - AllowedHeaders: ['*'], - MaxAgeSeconds: 3000 }, - ] }; - + const sampleCors = { + CORSRules: [ + { + AllowedMethods: ['PUT', 'POST', 'DELETE'], + AllowedOrigins: ['http://www.example.com'], + AllowedHeaders: ['*'], + MaxAgeSeconds: 3000, + ExposeHeaders: ['x-amz-server-side-encryption'], + }, + { AllowedMethods: ['GET'], AllowedOrigins: ['*'], AllowedHeaders: ['*'], MaxAgeSeconds: 3000 }, + ], + }; + before(async () => { await s3.send(new CreateBucketCommand({ Bucket: bucketName })); - await s3.send(new PutBucketCorsCommand({ - Bucket: bucketName, - CORSConfiguration: sampleCors, - })); + await s3.send( + new PutBucketCorsCommand({ + Bucket: bucketName, + CORSConfiguration: sampleCors, + }), + ); }); it('should return cors configuration successfully', async () => { @@ -46,45 +51,50 @@ describe('GET bucket cors', () => { describe('mixed case for AllowedHeader', () => { const testValue = 'tEsTvAlUe'; - const sampleCors = { CORSRules: [ - { AllowedMethods: ['PUT', 'POST', 'DELETE'], - AllowedOrigins: ['http://www.example.com'], - AllowedHeaders: [testValue] }, - ] }; - + const sampleCors = { + CORSRules: [ + { + AllowedMethods: ['PUT', 'POST', 'DELETE'], + AllowedOrigins: ['http://www.example.com'], + AllowedHeaders: [testValue], + }, + ], + }; + before(async () => { await s3.send(new CreateBucketCommand({ Bucket: bucketName })); - await s3.send(new PutBucketCorsCommand({ - Bucket: bucketName, - CORSConfiguration: sampleCors, - })); + await s3.send( + new PutBucketCorsCommand({ + Bucket: bucketName, + CORSConfiguration: sampleCors, + }), + ); }); it('should be preserved when putting / getting cors resource', async () => { const data = await s3.send(new GetBucketCorsCommand({ Bucket: bucketName })); - assert.deepStrictEqual(data.CORSRules[0].AllowedHeaders, - sampleCors.CORSRules[0].AllowedHeaders); + assert.deepStrictEqual(data.CORSRules[0].AllowedHeaders, sampleCors.CORSRules[0].AllowedHeaders); }); }); describe('uppercase for AllowedMethod', () => { - const sampleCors = { CORSRules: [ - { AllowedMethods: ['PUT', 'POST', 'DELETE'], - AllowedOrigins: ['http://www.example.com'] }, - ] }; - + const sampleCors = { + CORSRules: [{ AllowedMethods: ['PUT', 'POST', 'DELETE'], AllowedOrigins: ['http://www.example.com'] }], + }; + before(async () => { await s3.send(new CreateBucketCommand({ Bucket: bucketName })); - await s3.send(new PutBucketCorsCommand({ - Bucket: bucketName, - CORSConfiguration: sampleCors, - })); + await s3.send( + new PutBucketCorsCommand({ + Bucket: bucketName, + CORSConfiguration: sampleCors, + }), + ); }); it('should be preserved when retrieving cors resource', async () => { const data = await s3.send(new GetBucketCorsCommand({ Bucket: bucketName })); - assert.deepStrictEqual(data.CORSRules[0].AllowedMethods, - sampleCors.CORSRules[0].AllowedMethods); + assert.deepStrictEqual(data.CORSRules[0].AllowedMethods, sampleCors.CORSRules[0].AllowedMethods); }); }); diff --git a/tests/functional/aws-node-sdk/test/bucket/getLocation.js b/tests/functional/aws-node-sdk/test/bucket/getLocation.js index fc847c14ac..47efd6a29e 100644 --- a/tests/functional/aws-node-sdk/test/bucket/getLocation.js +++ b/tests/functional/aws-node-sdk/test/bucket/getLocation.js @@ -6,9 +6,7 @@ const BucketUtility = require('../../lib/utility/bucket-util'); const getConfig = require('../support/config'); const { config } = require('../../../../../lib/Config'); -const { - LOCATION_NAME_DMF, -} = require('../../../../constants'); +const { LOCATION_NAME_DMF } = require('../../../../constants'); const bucketName = 'testgetlocationbucket'; @@ -22,8 +20,7 @@ describeSkipAWS('GET bucket location ', () => { const otherAccountBucketUtility = new BucketUtility('lisa', {}); const otherAccountS3 = otherAccountBucketUtility.s3; const locationConstraints = config.locationConstraints; - Object.keys(locationConstraints).forEach( - location => { + Object.keys(locationConstraints).forEach(location => { if (location === 'us-east-1') { // if location is us-east-1 should return empty string // see next test. @@ -38,16 +35,19 @@ describeSkipAWS('GET bucket location ', () => { return; } describe(`with location: ${location}`, () => { - before(() => s3.send(new CreateBucketCommand({ - Bucket: bucketName, - CreateBucketConfiguration: { - LocationConstraint: location, - }, - }))); + before(() => + s3.send( + new CreateBucketCommand({ + Bucket: bucketName, + CreateBucketConfiguration: { + LocationConstraint: location, + }, + }), + ), + ); after(() => bucketUtil.deleteOne(bucketName)); - it(`should return location configuration: ${location} ` + - 'successfully', async () => { + it(`should return location configuration: ${location} ` + 'successfully', async () => { const data = await s3.send(new GetBucketLocationCommand({ Bucket: bucketName })); assert.deepStrictEqual(data.LocationConstraint, location); }); @@ -55,14 +55,18 @@ describeSkipAWS('GET bucket location ', () => { }); describe('with location us-east-1', () => { - before(() => s3.send(new CreateBucketCommand({ - Bucket: bucketName, - CreateBucketConfiguration: { - LocationConstraint: 'us-east-1', - }, - }))); + before(() => + s3.send( + new CreateBucketCommand({ + Bucket: bucketName, + CreateBucketConfiguration: { + LocationConstraint: 'us-east-1', + }, + }), + ), + ); afterEach(() => bucketUtil.deleteOne(bucketName)); - + it('should return empty location', async () => { const data = await s3.send(new GetBucketLocationCommand({ Bucket: bucketName })); // SDK v3 returns undefined for us-east-1, normalize to empty string for comparison @@ -74,20 +78,19 @@ describeSkipAWS('GET bucket location ', () => { describe('without location configuration', () => { after(() => { process.stdout.write('Deleting bucket\n'); - return bucketUtil.deleteOne(bucketName) - .catch(err => { - process.stdout.write(`Error in after: ${err}\n`); - throw err; - }); + return bucketUtil.deleteOne(bucketName).catch(err => { + process.stdout.write(`Error in after: ${err}\n`); + throw err; + }); }); it('should return request endpoint as location', async () => { process.stdout.write('Creating bucket'); await s3.send(new CreateBucketCommand({ Bucket: bucketName })); - + // In SDK v3, we need to get the endpoint from the client config let host = '127.0.0.1'; - + if (clientConfig.endpoint) { try { const url = new URL(clientConfig.endpoint); @@ -97,20 +100,20 @@ describeSkipAWS('GET bucket location ', () => { host = clientConfig.endpoint; } } - + let endpoint = config.restEndpoints[host]; // s3 actually returns '' for us-east-1 if (endpoint === 'us-east-1') { endpoint = ''; } - + const data = await s3.send(new GetBucketLocationCommand({ Bucket: bucketName })); - + // S3C backend has 'dc-1' as default location constraint // Other backends use endpoint-based location const isS3C = process.env.S3BACKEND === 's3c'; const expectedLocation = isS3C ? 'dc-1' : endpoint; - + const actualLocation = data.LocationConstraint || ''; const normalizedExpected = expectedLocation || ''; assert.strictEqual(actualLocation, normalizedExpected); @@ -118,12 +121,16 @@ describeSkipAWS('GET bucket location ', () => { }); describe('with location configuration', () => { - before(() => s3.send(new CreateBucketCommand({ - Bucket: bucketName, - CreateBucketConfiguration: { - LocationConstraint: 'us-east-1', - }, - }))); + before(() => + s3.send( + new CreateBucketCommand({ + Bucket: bucketName, + CreateBucketConfiguration: { + LocationConstraint: 'us-east-1', + }, + }), + ), + ); after(() => bucketUtil.deleteOne(bucketName)); it('should return AccessDenied if user is not bucket owner', async () => { diff --git a/tests/functional/aws-node-sdk/test/bucket/getWebsite.js b/tests/functional/aws-node-sdk/test/bucket/getWebsite.js index 766235d37c..458919cab3 100644 --- a/tests/functional/aws-node-sdk/test/bucket/getWebsite.js +++ b/tests/functional/aws-node-sdk/test/bucket/getWebsite.js @@ -1,9 +1,11 @@ const assert = require('assert'); -const { S3Client, +const { + S3Client, CreateBucketCommand, GetBucketWebsiteCommand, PutBucketWebsiteCommand, - DeleteBucketCommand } = require('@aws-sdk/client-s3'); + DeleteBucketCommand, +} = require('@aws-sdk/client-s3'); const withV4 = require('../support/withV4'); const getConfig = require('../support/config'); @@ -32,15 +34,17 @@ describe('GET bucket website', () => { const s3Config = getConfig('default', sigCfg); const s3 = new S3Client(s3Config); - afterEach(() => s3.send(new DeleteBucketCommand({ Bucket: bucketName }))); + afterEach(() => s3.send(new DeleteBucketCommand({ Bucket: bucketName }))); describe('with existing bucket configuration', () => { before(async () => { await s3.send(new CreateBucketCommand({ Bucket: bucketName })); - await s3.send(new PutBucketWebsiteCommand({ - Bucket: bucketName, - WebsiteConfiguration: config, - })); + await s3.send( + new PutBucketWebsiteCommand({ + Bucket: bucketName, + WebsiteConfiguration: config, + }), + ); }); it('should return bucket website xml successfully', async () => { diff --git a/tests/functional/aws-node-sdk/test/bucket/head.js b/tests/functional/aws-node-sdk/test/bucket/head.js index af8d18f854..64c28fcbd0 100644 --- a/tests/functional/aws-node-sdk/test/bucket/head.js +++ b/tests/functional/aws-node-sdk/test/bucket/head.js @@ -4,7 +4,6 @@ const { S3Client, HeadBucketCommand } = require('@aws-sdk/client-s3'); const withV4 = require('../support/withV4'); const getConfig = require('../support/config'); - describe('HEAD bucket', () => { withV4(sigCfg => { let s3; @@ -14,16 +13,14 @@ describe('HEAD bucket', () => { s3 = new S3Client(config); }); - it('should return an error to a head request without a ' + - 'bucket name', - async () => { - try { - await s3.send(new HeadBucketCommand({ Bucket: '' })); - assert.fail('Expected failure but got success'); - } catch (err) { - assert.strictEqual(err.$metadata.httpStatusCode, 405); - assert.strictEqual(err.name, 'Unknown'); - } - }); + it('should return an error to a head request without a ' + 'bucket name', async () => { + try { + await s3.send(new HeadBucketCommand({ Bucket: '' })); + assert.fail('Expected failure but got success'); + } catch (err) { + assert.strictEqual(err.$metadata.httpStatusCode, 405); + assert.strictEqual(err.name, 'Unknown'); + } + }); }); }); diff --git a/tests/functional/aws-node-sdk/test/bucket/listingCornerCases.js b/tests/functional/aws-node-sdk/test/bucket/listingCornerCases.js index 3f86f88b53..975a5f69af 100644 --- a/tests/functional/aws-node-sdk/test/bucket/listingCornerCases.js +++ b/tests/functional/aws-node-sdk/test/bucket/listingCornerCases.js @@ -1,11 +1,13 @@ -const { S3Client, +const { + S3Client, CreateBucketCommand, PutObjectCommand, DeleteObjectCommand, DeleteBucketCommand, ListObjectsCommand, ListObjectsV2Command, - PutBucketVersioningCommand } = require('@aws-sdk/client-s3'); + PutBucketVersioningCommand, +} = require('@aws-sdk/client-s3'); const assert = require('assert'); const getConfig = require('../support/config'); @@ -60,7 +62,7 @@ const allKeys = objects.map(obj => obj.Key); describe('Listing corner cases tests', () => { let s3; - + before(async () => { const config = getConfig('default', { signatureVersion: 'v4' }); s3 = new S3Client(config); @@ -83,17 +85,19 @@ describe('Listing corner cases tests', () => { Marker: '', MaxKeys: 1000, Name: Bucket, - Prefix: '' + Prefix: '', }); assert.strictEqual($metadata.httpStatusCode, 200); }); it('should list with valid marker', async () => { - const { $metadata, ...data } = await s3.send(new ListObjectsCommand({ - Bucket, - Delimiter: '/', - Marker: 'notes/summer/1.txt', - })); + const { $metadata, ...data } = await s3.send( + new ListObjectsCommand({ + Bucket, + Delimiter: '/', + Marker: 'notes/summer/1.txt', + }), + ); cutAttributes(data); assert.deepStrictEqual(data, { Delimiter: '/', @@ -101,17 +105,19 @@ describe('Listing corner cases tests', () => { Marker: 'notes/summer/1.txt', MaxKeys: 1000, Name: Bucket, - Prefix: '' + Prefix: '', }); assert.strictEqual($metadata.httpStatusCode, 200); }); it('should list with unexpected marker', async () => { - const { $metadata, ...data } = await s3.send(new ListObjectsCommand({ - Bucket, - Delimiter: '/', - Marker: 'zzzz', - })); + const { $metadata, ...data } = await s3.send( + new ListObjectsCommand({ + Bucket, + Delimiter: '/', + Marker: 'zzzz', + }), + ); assert.deepStrictEqual(data, { IsTruncated: false, Marker: 'zzzz', @@ -124,12 +130,14 @@ describe('Listing corner cases tests', () => { }); it('should list with unexpected marker and prefix', async () => { - const { $metadata, ...data } = await s3.send(new ListObjectsCommand({ - Bucket, - Delimiter: '/', - Marker: 'notes/summer0', - Prefix: 'notes/summer/', - })); + const { $metadata, ...data } = await s3.send( + new ListObjectsCommand({ + Bucket, + Delimiter: '/', + Marker: 'notes/summer0', + Prefix: 'notes/summer/', + }), + ); assert.deepStrictEqual(data, { IsTruncated: false, Marker: 'notes/summer0', @@ -142,27 +150,31 @@ describe('Listing corner cases tests', () => { }); it('should list with MaxKeys', async () => { - const { $metadata, ...data } = await s3.send(new ListObjectsCommand({ - Bucket, - MaxKeys: 3, - })); + const { $metadata, ...data } = await s3.send( + new ListObjectsCommand({ + Bucket, + MaxKeys: 3, + }), + ); cutAttributes(data); assert.deepStrictEqual(data, { Contents: objects.slice(0, 3).map(obj => obj.Key), IsTruncated: true, Marker: '', - MaxKeys: 3, + MaxKeys: 3, Name: Bucket, - Prefix: '' + Prefix: '', }); assert.strictEqual($metadata.httpStatusCode, 200); }); it('should list with big MaxKeys', async () => { - const { $metadata, ...data } = await s3.send(new ListObjectsCommand({ - Bucket, - MaxKeys: 15000, - })); + const { $metadata, ...data } = await s3.send( + new ListObjectsCommand({ + Bucket, + MaxKeys: 15000, + }), + ); cutAttributes(data); assert.deepStrictEqual(data, { Contents: allKeys, @@ -170,16 +182,18 @@ describe('Listing corner cases tests', () => { Marker: '', MaxKeys: 15000, Name: Bucket, - Prefix: '' + Prefix: '', }); assert.strictEqual($metadata.httpStatusCode, 200); }); it('should list with delimiter', async () => { - const { $metadata, ...data } = await s3.send(new ListObjectsCommand({ - Bucket, - Delimiter: '/', - })); + const { $metadata, ...data } = await s3.send( + new ListObjectsCommand({ + Bucket, + Delimiter: '/', + }), + ); cutAttributes(data); assert.deepStrictEqual(data, { Contents: [objects[0].Key], @@ -189,16 +203,18 @@ describe('Listing corner cases tests', () => { Marker: '', MaxKeys: 1000, Name: Bucket, - Prefix: '' + Prefix: '', }); assert.strictEqual($metadata.httpStatusCode, 200); }); it('should list with long delimiter', async () => { - const { $metadata, ...data } = await s3.send(new ListObjectsCommand({ - Bucket, - Delimiter: 'notes/summer', - })); + const { $metadata, ...data } = await s3.send( + new ListObjectsCommand({ + Bucket, + Delimiter: 'notes/summer', + }), + ); cutAttributes(data); assert.deepStrictEqual(data, { Marker: '', @@ -214,11 +230,13 @@ describe('Listing corner cases tests', () => { }); it('should list with delimiter and prefix related to #147', async () => { - const { $metadata, ...data } = await s3.send(new ListObjectsCommand({ - Bucket, - Delimiter: '/', - Prefix: 'notes/', - })); + const { $metadata, ...data } = await s3.send( + new ListObjectsCommand({ + Bucket, + Delimiter: '/', + Prefix: 'notes/', + }), + ); cutAttributes(data); assert.deepStrictEqual(data, { Marker: '', @@ -228,22 +246,20 @@ describe('Listing corner cases tests', () => { Prefix: 'notes/', Delimiter: '/', MaxKeys: 1000, - CommonPrefixes: [ - 'notes/spring/', - 'notes/summer/', - 'notes/zaphod/', - ], + CommonPrefixes: ['notes/spring/', 'notes/summer/', 'notes/zaphod/'], }); assert.strictEqual($metadata.httpStatusCode, 200); }); it('should list with prefix and marker related to #147', async () => { - const { $metadata, ...data } = await s3.send(new ListObjectsCommand({ - Bucket, - Delimiter: '/', - Prefix: 'notes/', - Marker: 'notes/year.txt', - })); + const { $metadata, ...data } = await s3.send( + new ListObjectsCommand({ + Bucket, + Delimiter: '/', + Prefix: 'notes/', + Marker: 'notes/year.txt', + }), + ); cutAttributes(data); assert.deepStrictEqual(data, { Marker: 'notes/year.txt', @@ -259,13 +275,15 @@ describe('Listing corner cases tests', () => { }); it('should list with all parameters 1 of 5', async () => { - const { $metadata, ...data } = await s3.send(new ListObjectsCommand({ - Bucket, - Delimiter: '/', - Prefix: 'notes/', - Marker: 'notes/', - MaxKeys: 1, - })); + const { $metadata, ...data } = await s3.send( + new ListObjectsCommand({ + Bucket, + Delimiter: '/', + Prefix: 'notes/', + Marker: 'notes/', + MaxKeys: 1, + }), + ); cutAttributes(data); assert.deepStrictEqual(data, { Marker: 'notes/', @@ -281,13 +299,15 @@ describe('Listing corner cases tests', () => { }); it('should list with all parameters 2 of 5', async () => { - const { $metadata, ...data } = await s3.send(new ListObjectsCommand({ - Bucket, - Delimiter: '/', - Prefix: 'notes/', - Marker: 'notes/spring/', - MaxKeys: 1, - })); + const { $metadata, ...data } = await s3.send( + new ListObjectsCommand({ + Bucket, + Delimiter: '/', + Prefix: 'notes/', + Marker: 'notes/spring/', + MaxKeys: 1, + }), + ); cutAttributes(data); assert.deepStrictEqual(data, { Marker: 'notes/spring/', @@ -303,13 +323,15 @@ describe('Listing corner cases tests', () => { }); it('should list with all parameters 3 of 5', async () => { - const { $metadata, ...data } = await s3.send(new ListObjectsCommand({ - Bucket, - Delimiter: '/', - Prefix: 'notes/', - Marker: 'notes/summer/', - MaxKeys: 1, - })); + const { $metadata, ...data } = await s3.send( + new ListObjectsCommand({ + Bucket, + Delimiter: '/', + Prefix: 'notes/', + Marker: 'notes/summer/', + MaxKeys: 1, + }), + ); cutAttributes(data); assert.deepStrictEqual(data, { Marker: 'notes/summer/', @@ -325,13 +347,15 @@ describe('Listing corner cases tests', () => { }); it('should list with all parameters 4 of 5', async () => { - const { $metadata, ...data } = await s3.send(new ListObjectsCommand({ - Bucket, - Delimiter: '/', - Prefix: 'notes/', - Marker: 'notes/year.txt', - MaxKeys: 1, - })); + const { $metadata, ...data } = await s3.send( + new ListObjectsCommand({ + Bucket, + Delimiter: '/', + Prefix: 'notes/', + Marker: 'notes/year.txt', + MaxKeys: 1, + }), + ); cutAttributes(data); assert.deepStrictEqual(data, { Marker: 'notes/year.txt', @@ -347,13 +371,15 @@ describe('Listing corner cases tests', () => { }); it('should list with all parameters 5 of 5', async () => { - const { $metadata, ...data } = await s3.send(new ListObjectsCommand({ - Bucket, - Delimiter: '/', - Prefix: 'notes/', - Marker: 'notes/yore.rs', - MaxKeys: 1, - })); + const { $metadata, ...data } = await s3.send( + new ListObjectsCommand({ + Bucket, + Delimiter: '/', + Prefix: 'notes/', + Marker: 'notes/yore.rs', + MaxKeys: 1, + }), + ); cutAttributes(data); assert.deepStrictEqual(data, { Marker: 'notes/yore.rs', @@ -368,18 +394,22 @@ describe('Listing corner cases tests', () => { }); it('should end listing on last common prefix', async () => { - await s3.send(new PutObjectCommand({ - Bucket, - Key: 'notes/zaphod/TheFourth.txt', - Body: '', - })); - const { $metadata, ...data } = await s3.send(new ListObjectsCommand({ - Bucket, - Delimiter: '/', - Prefix: 'notes/', - Marker: 'notes/yore.rs', - MaxKeys: 1, - })); + await s3.send( + new PutObjectCommand({ + Bucket, + Key: 'notes/zaphod/TheFourth.txt', + Body: '', + }), + ); + const { $metadata, ...data } = await s3.send( + new ListObjectsCommand({ + Bucket, + Delimiter: '/', + Prefix: 'notes/', + Marker: 'notes/yore.rs', + MaxKeys: 1, + }), + ); cutAttributes(data); assert.deepStrictEqual(data, { IsTruncated: false, @@ -396,49 +426,62 @@ describe('Listing corner cases tests', () => { it('should not list DeleteMarkers for version suspended buckets', async () => { const obj = { name: 'testDeleteMarker.txt', value: 'foo' }; const bucketName = `bucket-test-delete-markers-not-listed${Date.now()}`; - + try { await s3.send(new CreateBucketCommand({ Bucket: bucketName })); - - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: { - Status: 'Suspended', - }, - })); - - await s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: obj.name, - Body: obj.value, - })); - + + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: { + Status: 'Suspended', + }, + }), + ); + + await s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: obj.name, + Body: obj.value, + }), + ); + const listRes1 = await s3.send(new ListObjectsV2Command({ Bucket: bucketName })); - assert.strictEqual(listRes1.Contents.some(c => c.Key === obj.name), true); - - const deleteRes = await s3.send(new DeleteObjectCommand({ - Bucket: bucketName, - Key: obj.name, - })); + assert.strictEqual( + listRes1.Contents.some(c => c.Key === obj.name), + true, + ); + + const deleteRes = await s3.send( + new DeleteObjectCommand({ + Bucket: bucketName, + Key: obj.name, + }), + ); assert.strictEqual(deleteRes.DeleteMarker, true); - + const listRes2 = await s3.send(new ListObjectsV2Command({ Bucket: bucketName })); assert.strictEqual(listRes2.Contents, undefined); - - await s3.send(new DeleteObjectCommand({ - Bucket: bucketName, - Key: obj.name, - VersionId: 'null' - })); - + + await s3.send( + new DeleteObjectCommand({ + Bucket: bucketName, + Key: obj.name, + VersionId: 'null', + }), + ); + await s3.send(new DeleteBucketCommand({ Bucket: bucketName })); } catch (err) { try { - await s3.send(new DeleteObjectCommand({ - Bucket: bucketName, - Key: obj.name, - VersionId: 'null' - })); + await s3.send( + new DeleteObjectCommand({ + Bucket: bucketName, + Key: obj.name, + VersionId: 'null', + }), + ); await s3.send(new DeleteBucketCommand({ Bucket: bucketName })); } catch { // Ignore cleanup errors diff --git a/tests/functional/aws-node-sdk/test/bucket/put.js b/tests/functional/aws-node-sdk/test/bucket/put.js index 5773adbca1..3ffcc811d7 100644 --- a/tests/functional/aws-node-sdk/test/bucket/put.js +++ b/tests/functional/aws-node-sdk/test/bucket/put.js @@ -12,9 +12,7 @@ const BucketUtility = require('../../lib/utility/bucket-util'); const withV4 = require('../support/withV4'); const configOfficial = require('../../../../../lib/Config').config; -const { - LOCATION_NAME_DMF, -} = require('../../../../constants'); +const { LOCATION_NAME_DMF } = require('../../../../constants'); const bucketName = 'bucketlocation'; @@ -25,7 +23,6 @@ const locationConstraints = configOfficial.locationConstraints; describe('PUT Bucket - AWS.S3.createBucket', () => { describe('When user is unauthorized', () => { - it('should return 403 and AccessDenied', async () => { const params = { Bucket: 'mybucket' }; try { @@ -45,44 +42,51 @@ describe('PUT Bucket - AWS.S3.createBucket', () => { before(() => { bucketUtil = new BucketUtility('default', sigCfg); }); - + describe('create bucket twice', () => { let testBucketName; - + beforeEach(() => { // Use unique bucket name for each test to avoid conflicts testBucketName = `${bucketName}-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`; - return bucketUtil.s3.send(new CreateBucketCommand({ - Bucket: testBucketName, - CreateBucketConfiguration: { - LocationConstraint: 'us-east-1', - }, - })); + return bucketUtil.s3.send( + new CreateBucketCommand({ + Bucket: testBucketName, + CreateBucketConfiguration: { + LocationConstraint: 'us-east-1', + }, + }), + ); }); - + afterEach(() => bucketUtil.s3.send(new DeleteBucketCommand({ Bucket: testBucketName }))); - - itSkipIfE2E('should return a 200 if no locationConstraints provided.', - () => bucketUtil.s3.send(new CreateBucketCommand({ Bucket: testBucketName }))); + + itSkipIfE2E('should return a 200 if no locationConstraints provided.', () => + bucketUtil.s3.send(new CreateBucketCommand({ Bucket: testBucketName })), + ); it('should return a 200 if us-east behavior', async () => { - const res = await bucketUtil.s3.send(new CreateBucketCommand({ - Bucket: testBucketName, - CreateBucketConfiguration: { - LocationConstraint: 'us-east-1', - }, - })); + const res = await bucketUtil.s3.send( + new CreateBucketCommand({ + Bucket: testBucketName, + CreateBucketConfiguration: { + LocationConstraint: 'us-east-1', + }, + }), + ); assert.strictEqual(res.$metadata.httpStatusCode, 200); }); - + it('should return a 409 if us-west behavior', async () => { try { - await bucketUtil.s3.send(new CreateBucketCommand({ - Bucket: testBucketName, - CreateBucketConfiguration: { - LocationConstraint: 'scality-us-west-1', - }, - })); + await bucketUtil.s3.send( + new CreateBucketCommand({ + Bucket: testBucketName, + CreateBucketConfiguration: { + LocationConstraint: 'scality-us-west-1', + }, + }), + ); assert.fail('Expected failure but got success'); } catch (error) { assert.strictEqual(error.name, 'BucketAlreadyOwnedByYou'); @@ -101,15 +105,13 @@ describe('PUT Bucket - AWS.S3.createBucket', () => { bucketUtil .createOne(bucketName) .then(() => { - const e = new Error('Expect failure in creation, ' + - 'but it succeeded'); + const e = new Error('Expect failure in creation, ' + 'but it succeeded'); return done(e); }) .catch(error => { assert.strictEqual(error.Code, expectedCode); - assert.strictEqual(error.$metadata.httpStatusCode, - expectedStatus); + assert.strictEqual(error.$metadata.httpStatusCode, expectedStatus); done(); }); }; @@ -133,45 +135,35 @@ describe('PUT Bucket - AWS.S3.createBucket', () => { testFn(shortName, done); }); - it('should return 403 if name is reserved (e.g., METADATA)', - done => { - const reservedName = 'METADATA'; - testFn(reservedName, done, 403, 'AccessDenied'); - }); + it('should return 403 if name is reserved (e.g., METADATA)', done => { + const reservedName = 'METADATA'; + testFn(reservedName, done, 403, 'AccessDenied'); + }); - itSkipIfAWS('should return 400 if name is longer than 63 chars', - done => { - const longName = 'x'.repeat(64); - testFn(longName, done); - } - ); + itSkipIfAWS('should return 400 if name is longer than 63 chars', done => { + const longName = 'x'.repeat(64); + testFn(longName, done); + }); - itSkipIfAWS('should return 400 if name is formatted as IP address', - done => { - const ipAddress = '192.168.5.4'; - testFn(ipAddress, done); - } - ); + itSkipIfAWS('should return 400 if name is formatted as IP address', done => { + const ipAddress = '192.168.5.4'; + testFn(ipAddress, done); + }); - itSkipIfAWS('should return 400 if name starts with period', - done => { - const invalidName = '.myawsbucket'; - testFn(invalidName, done); - } - ); + itSkipIfAWS('should return 400 if name starts with period', done => { + const invalidName = '.myawsbucket'; + testFn(invalidName, done); + }); it('should return 400 if name ends with period', done => { const invalidName = 'myawsbucket.'; testFn(invalidName, done); }); - itSkipIfAWS( - 'should return 400 if name has two period between labels', - done => { - const invalidName = 'my..examplebucket'; - testFn(invalidName, done); - } - ); + itSkipIfAWS('should return 400 if name has two period between labels', done => { + const invalidName = 'my..examplebucket'; + testFn(invalidName, done); + }); it('should return 400 if name has special chars', done => { const invalidName = 'my.#s3bucket'; @@ -181,71 +173,89 @@ describe('PUT Bucket - AWS.S3.createBucket', () => { describe('bucket creation success', () => { function _test(name, done) { - bucketUtil.s3.send(new CreateBucketCommand({ Bucket: name })) + bucketUtil.s3 + .send(new CreateBucketCommand({ Bucket: name })) .then(res => { assert(res.Location, 'No Location in response'); - assert.deepStrictEqual(res.Location, `/${name}`, - 'Wrong Location header'); + assert.deepStrictEqual(res.Location, `/${name}`, 'Wrong Location header'); return bucketUtil.deleteOne(name); }) .then(() => done()) .catch(done); } - it('should create bucket if name is valid', done => - _test('scality-very-valid-bucket-name', done)); + it('should create bucket if name is valid', done => _test('scality-very-valid-bucket-name', done)); - it('should create bucket if name is some prefix and an IP address', - done => _test('prefix-192.168.5.4', done)); + it('should create bucket if name is some prefix and an IP address', done => + _test('prefix-192.168.5.4', done)); - it('should create bucket if name is an IP address with some suffix', - done => _test('192.168.5.4-suffix', done)); + it('should create bucket if name is an IP address with some suffix', done => + _test('192.168.5.4-suffix', done)); }); describe('bucket creation success with object lock', () => { function _testObjectLockEnabled(name, done) { - bucketUtil.s3.send(new CreateBucketCommand({ - Bucket: name, - ObjectLockEnabledForBucket: true, - })).then(res => { - assert.strictEqual(res.Location, `/${name}`, 'Wrong Location header'); - return bucketUtil.s3.send(new GetObjectLockConfigurationCommand({ Bucket: name })); - }).then(res => { - assert.deepStrictEqual(res.ObjectLockConfiguration, - { ObjectLockEnabled: 'Enabled' }); - return bucketUtil.deleteOne(name); - }).then(() => done()).catch(done); + bucketUtil.s3 + .send( + new CreateBucketCommand({ + Bucket: name, + ObjectLockEnabledForBucket: true, + }), + ) + .then(res => { + assert.strictEqual(res.Location, `/${name}`, 'Wrong Location header'); + return bucketUtil.s3.send(new GetObjectLockConfigurationCommand({ Bucket: name })); + }) + .then(res => { + assert.deepStrictEqual(res.ObjectLockConfiguration, { ObjectLockEnabled: 'Enabled' }); + return bucketUtil.deleteOne(name); + }) + .then(() => done()) + .catch(done); } function _testObjectLockDisabled(name, done) { - bucketUtil.s3.send(new CreateBucketCommand({ - Bucket: name, - ObjectLockEnabledForBucket: false, - })).then(res => { - assert(res.Location, 'No Location in response'); - assert.strictEqual(res.Location, `/${name}`, 'Wrong Location header'); - return bucketUtil.s3.send(new GetObjectLockConfigurationCommand({ Bucket: name })); - }).catch(err => { - assert.strictEqual(err.name, 'ObjectLockConfigurationNotFoundError'); - return bucketUtil.deleteOne(name); - }).then(() => done()).catch(done); + bucketUtil.s3 + .send( + new CreateBucketCommand({ + Bucket: name, + ObjectLockEnabledForBucket: false, + }), + ) + .then(res => { + assert(res.Location, 'No Location in response'); + assert.strictEqual(res.Location, `/${name}`, 'Wrong Location header'); + return bucketUtil.s3.send(new GetObjectLockConfigurationCommand({ Bucket: name })); + }) + .catch(err => { + assert.strictEqual(err.name, 'ObjectLockConfigurationNotFoundError'); + return bucketUtil.deleteOne(name); + }) + .then(() => done()) + .catch(done); } function _testVersioning(name, done) { - bucketUtil.s3.send(new CreateBucketCommand({ - Bucket: name, - ObjectLockEnabledForBucket: true, - })).then(res => { - assert(res.Location, 'No Location in response'); - assert.strictEqual(res.Location, `/${name}`, 'Wrong Location header'); - return bucketUtil.s3.send(new GetBucketVersioningCommand({ Bucket: name })); - }).then(res => { - assert.strictEqual(res.Status, 'Enabled'); - assert.strictEqual(res.MFADelete, 'Disabled'); - return bucketUtil.deleteOne(name); - }).then(() => done()).catch(done); + bucketUtil.s3 + .send( + new CreateBucketCommand({ + Bucket: name, + ObjectLockEnabledForBucket: true, + }), + ) + .then(res => { + assert(res.Location, 'No Location in response'); + assert.strictEqual(res.Location, `/${name}`, 'Wrong Location header'); + return bucketUtil.s3.send(new GetBucketVersioningCommand({ Bucket: name })); + }) + .then(res => { + assert.strictEqual(res.Status, 'Enabled'); + assert.strictEqual(res.MFADelete, 'Disabled'); + return bucketUtil.deleteOne(name); + }) + .then(() => done()) + .catch(done); } - it('should create bucket without error', done => - _testObjectLockEnabled('bucket-with-object-lock', done)); + it('should create bucket without error', done => _testObjectLockEnabled('bucket-with-object-lock', done)); it('should create bucket with versioning enabled by default', done => _testVersioning('bucket-with-object-lock', done)); @@ -254,98 +264,121 @@ describe('PUT Bucket - AWS.S3.createBucket', () => { _testObjectLockDisabled('bucket-without-object-lock', done)); }); - Object.keys(locationConstraints).forEach( - location => { - describeSkipAWS(`bucket creation with location: ${location}`, - () => { + Object.keys(locationConstraints).forEach(location => { + describeSkipAWS(`bucket creation with location: ${location}`, () => { after(done => { bucketUtil.deleteOne(bucketName).finally(done); }); it(`should create bucket with location: ${location}`, done => { - bucketUtil.s3.send(new CreateBucketCommand({ - Bucket: bucketName, - CreateBucketConfiguration: { - LocationConstraint: location, - }, - })).then(() => { - done(); - }).catch(err => { - if (location === LOCATION_NAME_DMF) { - assert.strictEqual( - err.name, - 'InvalidLocationConstraint' - ); - assert.strictEqual(err.$metadata.httpStatusCode, 400); - } - done(); - }); + bucketUtil.s3 + .send( + new CreateBucketCommand({ + Bucket: bucketName, + CreateBucketConfiguration: { + LocationConstraint: location, + }, + }), + ) + .then(() => { + done(); + }) + .catch(err => { + if (location === LOCATION_NAME_DMF) { + assert.strictEqual(err.name, 'InvalidLocationConstraint'); + assert.strictEqual(err.$metadata.httpStatusCode, 400); + } + done(); + }); }); }); }); describe('bucket creation with invalid location', () => { it('should return errors InvalidLocationConstraint', done => { - bucketUtil.s3.send(new CreateBucketCommand({ - Bucket: bucketName, - CreateBucketConfiguration: { - LocationConstraint: 'coco', - }, - })).catch(err => { - assert.strictEqual( - err.name, - 'InvalidLocationConstraint' - ); - assert.strictEqual(err.$metadata.httpStatusCode, 400); - done(); - }); + bucketUtil.s3 + .send( + new CreateBucketCommand({ + Bucket: bucketName, + CreateBucketConfiguration: { + LocationConstraint: 'coco', + }, + }), + ) + .catch(err => { + assert.strictEqual(err.name, 'InvalidLocationConstraint'); + assert.strictEqual(err.$metadata.httpStatusCode, 400); + done(); + }); }); it('should return error InvalidLocationConstraint for location constraint dmf', done => { - bucketUtil.s3.send(new CreateBucketCommand({ - Bucket: bucketName, - CreateBucketConfiguration: { - LocationConstraint: LOCATION_NAME_DMF, - }, - })).catch(err => { - assert.strictEqual( - err.name, - 'InvalidLocationConstraint' - ); - assert.strictEqual(err.$metadata.httpStatusCode, 400); - done(); - }); + bucketUtil.s3 + .send( + new CreateBucketCommand({ + Bucket: bucketName, + CreateBucketConfiguration: { + LocationConstraint: LOCATION_NAME_DMF, + }, + }), + ) + .catch(err => { + assert.strictEqual(err.name, 'InvalidLocationConstraint'); + assert.strictEqual(err.$metadata.httpStatusCode, 400); + done(); + }); }); }); describe('bucket creation with ingestion location', () => { - after(() => bucketUtil.s3.send(new DeleteBucketCommand({ Bucket: bucketName }))); - + after(() => bucketUtil.s3.send(new DeleteBucketCommand({ Bucket: bucketName }))); + it('should create bucket with location and ingestion', done => { - async.waterfall([ - next => bucketUtil.s3.send(new CreateBucketCommand({ - Bucket: bucketName, - CreateBucketConfiguration: { - LocationConstraint: 'us-east-2:ingest', - }, - })).then(res => { - assert.strictEqual(res.Location, `/${bucketName}`); - next(); - }).catch(next), - - next => bucketUtil.s3.send(new GetBucketLocationCommand({ - Bucket: bucketName, - })).then(res => { - assert.strictEqual(res.LocationConstraint, 'us-east-2'); - next(); - }).catch(next), - - next => bucketUtil.s3.send(new GetBucketVersioningCommand({ - Bucket: bucketName, - })).then(res => { - assert.strictEqual(res.Status, 'Enabled'); - next(); - }).catch(next), - ], done); + async.waterfall( + [ + next => + bucketUtil.s3 + .send( + new CreateBucketCommand({ + Bucket: bucketName, + CreateBucketConfiguration: { + LocationConstraint: 'us-east-2:ingest', + }, + }), + ) + .then(res => { + assert.strictEqual(res.Location, `/${bucketName}`); + next(); + }) + .catch(next), + + next => + bucketUtil.s3 + .send( + new GetBucketLocationCommand({ + Bucket: bucketName, + }), + ) + .then(res => { + assert.strictEqual(res.LocationConstraint, 'us-east-2'); + next(); + }) + .catch(next), + + next => + bucketUtil.s3 + .send( + new GetBucketVersioningCommand({ + Bucket: bucketName, + }), + ) + .then(res => { + assert.strictEqual(res.Status, 'Enabled'); + next(); + }) + .catch(next), + ], + done, + ); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/bucket/putAcl.js b/tests/functional/aws-node-sdk/test/bucket/putAcl.js index dc482d7de0..98c9d1535f 100644 --- a/tests/functional/aws-node-sdk/test/bucket/putAcl.js +++ b/tests/functional/aws-node-sdk/test/bucket/putAcl.js @@ -1,9 +1,11 @@ const assert = require('assert'); -const { S3Client, +const { + S3Client, CreateBucketCommand, DeleteBucketCommand, PutBucketAclCommand, - GetBucketAclCommand } = require('@aws-sdk/client-s3'); + GetBucketAclCommand, +} = require('@aws-sdk/client-s3'); const getConfig = require('../support/config'); const withV4 = require('../support/withV4'); const BucketUtility = require('../../lib/utility/bucket-util'); @@ -53,8 +55,7 @@ describe('aws-node-sdk test bucket put acl', () => { throw new Error('accepted xml body larger than 512 KB'); } catch (error) { assert.strictEqual(error.$metadata.httpStatusCode, 400); - assert.strictEqual( - error.name, 'InvalidRequest'); + assert.strictEqual(error.name, 'InvalidRequest'); } }); }); @@ -68,26 +69,30 @@ describe('PUT Bucket ACL', () => { afterEach(() => bucketUtil.deleteOne(bucketName)); - it('should set multiple ACL permissions with same grantee specified' + - 'using email', async () => { - await s3.send(new PutBucketAclCommand({ - Bucket: bucketName, - GrantRead: 'emailAddress=sampleaccount1@sampling.com', - GrantWrite: 'emailAddress=sampleaccount1@sampling.com', - })); - const res = await s3.send(new GetBucketAclCommand({ - Bucket: bucketName, - })); + it('should set multiple ACL permissions with same grantee specified' + 'using email', async () => { + await s3.send( + new PutBucketAclCommand({ + Bucket: bucketName, + GrantRead: 'emailAddress=sampleaccount1@sampling.com', + GrantWrite: 'emailAddress=sampleaccount1@sampling.com', + }), + ); + const res = await s3.send( + new GetBucketAclCommand({ + Bucket: bucketName, + }), + ); assert.strictEqual(res.Grants.length, 2); }); - it('should return InvalidArgument if invalid grantee ' + - 'user ID provided in ACL header request', async () => { + it('should return InvalidArgument if invalid grantee ' + 'user ID provided in ACL header request', async () => { try { - await s3.send(new PutBucketAclCommand({ - Bucket: bucketName, - GrantRead: 'id=invalidUserID' - })); + await s3.send( + new PutBucketAclCommand({ + Bucket: bucketName, + GrantRead: 'id=invalidUserID', + }), + ); throw new Error('Expected InvalidArgument error'); } catch (err) { assert.strictEqual(err.$metadata.httpStatusCode, 400); @@ -95,27 +100,28 @@ describe('PUT Bucket ACL', () => { } }); - it('should return InvalidArgument if invalid grantee ' + - 'user ID provided in ACL request body', async () => { + it('should return InvalidArgument if invalid grantee ' + 'user ID provided in ACL request body', async () => { try { - await s3.send(new PutBucketAclCommand({ - Bucket: bucketName, - AccessControlPolicy: { - Grants: [ - { - Grantee: { - Type: 'CanonicalUser', - ID: 'invalidUserID', + await s3.send( + new PutBucketAclCommand({ + Bucket: bucketName, + AccessControlPolicy: { + Grants: [ + { + Grantee: { + Type: 'CanonicalUser', + ID: 'invalidUserID', + }, + Permission: 'WRITE_ACP', }, - Permission: 'WRITE_ACP', - }], - Owner: { - DisplayName: 'Bart', - ID: '79a59df900b949e55d96a1e698fbace' + - 'dfd6e09d98eacf8f8d5218e7cd47ef2be', + ], + Owner: { + DisplayName: 'Bart', + ID: '79a59df900b949e55d96a1e698fbace' + 'dfd6e09d98eacf8f8d5218e7cd47ef2be', + }, }, - }, - })); + }), + ); throw new Error('Expected InvalidArgument error'); } catch (err) { assert.strictEqual(err.$metadata.httpStatusCode, 400); diff --git a/tests/functional/aws-node-sdk/test/bucket/putBucketLifecycle.js b/tests/functional/aws-node-sdk/test/bucket/putBucketLifecycle.js index ecec515963..19941bb585 100644 --- a/tests/functional/aws-node-sdk/test/bucket/putBucketLifecycle.js +++ b/tests/functional/aws-node-sdk/test/bucket/putBucketLifecycle.js @@ -1,9 +1,11 @@ const assert = require('assert'); const { errors } = require('arsenal'); -const { S3Client, +const { + S3Client, CreateBucketCommand, DeleteBucketCommand, - PutBucketLifecycleConfigurationCommand } = require('@aws-sdk/client-s3'); + PutBucketLifecycleConfigurationCommand, +} = require('@aws-sdk/client-s3'); const getConfig = require('../support/config'); const BucketUtility = require('../../lib/utility/bucket-util'); @@ -26,11 +28,17 @@ function assertError(err, expectedErr) { if (expectedErr === null) { assert.strictEqual(err, null, `expected no error but got '${err}'`); } else { - assert.strictEqual(err.name, expectedErr, 'incorrect error response ' + - `code: should be '${expectedErr}' but got '${err.name}'`); - assert.strictEqual(err.$metadata.httpStatusCode, errors[expectedErr].code, + assert.strictEqual( + err.name, + expectedErr, + 'incorrect error response ' + `code: should be '${expectedErr}' but got '${err.name}'`, + ); + assert.strictEqual( + err.$metadata.httpStatusCode, + errors[expectedErr].code, 'incorrect error status code: should be ' + - `${errors[expectedErr].code}, but got '${err.$metadata.httpStatusCode}'`); + `${errors[expectedErr].code}, but got '${err.$metadata.httpStatusCode}'`, + ); } } @@ -89,40 +97,42 @@ describe('aws-sdk test put bucket lifecycle', () => { const params = getLifecycleParams(); await s3.send(new PutBucketLifecycleConfigurationCommand(params)); }); - - it('should not allow lifecycle configuration with duplicated rule id ' + - 'and with Origin header set', async () => { - const origin = 'http://www.allowedwebsite.com'; - const lifecycleConfig = { - Rules: [expirationRule, expirationRule], - }; - const params = { - Bucket: bucket, - LifecycleConfiguration: lifecycleConfig, - }; - const clientConfig = getConfig('default', { signatureVersion: 'v4' }); - const clientWithOrigin = new S3Client({ - ...clientConfig, - requestHandler: { - handle: async request => { - if (!request.headers) { + it( + 'should not allow lifecycle configuration with duplicated rule id ' + 'and with Origin header set', + async () => { + const origin = 'http://www.allowedwebsite.com'; + const lifecycleConfig = { + Rules: [expirationRule, expirationRule], + }; + const params = { + Bucket: bucket, + LifecycleConfiguration: lifecycleConfig, + }; + + const clientConfig = getConfig('default', { signatureVersion: 'v4' }); + const clientWithOrigin = new S3Client({ + ...clientConfig, + requestHandler: { + handle: async request => { + if (!request.headers) { + // eslint-disable-next-line no-param-reassign + request.headers = {}; + } // eslint-disable-next-line no-param-reassign - request.headers = {}; - } - // eslint-disable-next-line no-param-reassign - request.headers.origin = origin; - return clientConfig.requestHandler.handle(request); - } + request.headers.origin = origin; + return clientConfig.requestHandler.handle(request); + }, + }, + }); + try { + await clientWithOrigin.send(new PutBucketLifecycleConfigurationCommand(params)); + throw new Error('Expected InvalidRequest error'); + } catch (err) { + assertError(err, 'InvalidRequest'); } - }); - try { - await clientWithOrigin.send(new PutBucketLifecycleConfigurationCommand(params)); - throw new Error('Expected InvalidRequest error'); - } catch (err) { - assertError(err, 'InvalidRequest'); - } - }); + }, + ); it('should not allow lifecycle config with no Status', async () => { const params = getLifecycleParams({ key: 'Status', value: '' }); @@ -155,8 +165,7 @@ describe('aws-sdk test put bucket lifecycle', () => { }); it('should not allow lifecycle config with ID longer than 255 char', async () => { - const params = - getLifecycleParams({ key: 'ID', value: 'a'.repeat(256) }); + const params = getLifecycleParams({ key: 'ID', value: 'a'.repeat(256) }); try { await s3.send(new PutBucketLifecycleConfigurationCommand(params)); throw new Error('Expected InvalidArgument error'); @@ -166,20 +175,17 @@ describe('aws-sdk test put bucket lifecycle', () => { }); it('should allow lifecycle config with Prefix length < 1024', async () => { - const params = - getLifecycleParams({ key: 'Prefix', value: 'a'.repeat(1023) }); + const params = getLifecycleParams({ key: 'Prefix', value: 'a'.repeat(1023) }); await s3.send(new PutBucketLifecycleConfigurationCommand(params)); }); it('should allow lifecycle config with Prefix length === 1024', async () => { - const params = - getLifecycleParams({ key: 'Prefix', value: 'a'.repeat(1024) }); + const params = getLifecycleParams({ key: 'Prefix', value: 'a'.repeat(1024) }); await s3.send(new PutBucketLifecycleConfigurationCommand(params)); }); it('should not allow lifecycle config with Prefix length > 1024', async () => { - const params = - getLifecycleParams({ key: 'Prefix', value: 'a'.repeat(1025) }); + const params = getLifecycleParams({ key: 'Prefix', value: 'a'.repeat(1025) }); try { await s3.send(new PutBucketLifecycleConfigurationCommand(params)); throw new Error('Expected InvalidRequest error'); @@ -202,8 +208,7 @@ describe('aws-sdk test put bucket lifecycle', () => { } }); - it('should not allow lifecycle config with Filter.And.Prefix length ' + - '> 1024', async () => { + it('should not allow lifecycle config with Filter.And.Prefix length ' + '> 1024', async () => { const params = getLifecycleParams({ key: 'Filter', value: { @@ -287,8 +292,7 @@ describe('aws-sdk test put bucket lifecycle', () => { }); it('should not allow lifecycle config with Prefix and Filter', async () => { - const params = getLifecycleParams( - { key: 'Filter', value: { Prefix: 'foo' } }); + const params = getLifecycleParams({ key: 'Filter', value: { Prefix: 'foo' } }); try { await s3.send(new PutBucketLifecycleConfigurationCommand(params)); throw new Error('Expected MalformedXML error'); @@ -310,7 +314,6 @@ describe('aws-sdk test put bucket lifecycle', () => { await s3.send(new PutBucketLifecycleConfigurationCommand(params)); }); - describe('with Rule.Filter not Rule.Prefix', () => { before(done => { expirationRule.Prefix = null; @@ -323,8 +326,7 @@ describe('aws-sdk test put bucket lifecycle', () => { }); it('should not allow config with And & Prefix', async () => { - const params = getLifecycleParams( - { key: 'Filter', value: { Prefix: 'foo', And: {} } }); + const params = getLifecycleParams({ key: 'Filter', value: { Prefix: 'foo', And: {} } }); try { await s3.send(new PutBucketLifecycleConfigurationCommand(params)); throw new Error('Expected MalformedXML error'); @@ -360,8 +362,7 @@ describe('aws-sdk test put bucket lifecycle', () => { }); it('should allow config with only Prefix', async () => { - const params = getLifecycleParams( - { key: 'Filter', value: { Prefix: 'foo' } }); + const params = getLifecycleParams({ key: 'Filter', value: { Prefix: 'foo' } }); await s3.send(new PutBucketLifecycleConfigurationCommand(params)); }); @@ -374,8 +375,7 @@ describe('aws-sdk test put bucket lifecycle', () => { }); it('should not allow config with And.Prefix & no And.Tags', async () => { - const params = getLifecycleParams( - { key: 'Filter', value: { And: { Prefix: 'foo' } } }); + const params = getLifecycleParams({ key: 'Filter', value: { And: { Prefix: 'foo' } } }); try { await s3.send(new PutBucketLifecycleConfigurationCommand(params)); throw new Error('Expected MalformedXML error'); @@ -400,9 +400,14 @@ describe('aws-sdk test put bucket lifecycle', () => { it('should allow config with And.Tags & no And.Prefix', async () => { const params = getLifecycleParams({ key: 'Filter', - value: { And: { Tags: - [{ Key: 'foo', Value: 'bar' }, - { Key: 'foo2', Value: 'bar2' }] } }, + value: { + And: { + Tags: [ + { Key: 'foo', Value: 'bar' }, + { Key: 'foo2', Value: 'bar2' }, + ], + }, + }, }); await s3.send(new PutBucketLifecycleConfigurationCommand(params)); }); @@ -410,9 +415,15 @@ describe('aws-sdk test put bucket lifecycle', () => { it('should allow config with And.Tags & And.Prefix', async () => { const params = getLifecycleParams({ key: 'Filter', - value: { And: { Prefix: 'foo', Tags: - [{ Key: 'foo', Value: 'bar' }, - { Key: 'foo2', Value: 'bar2' }] } }, + value: { + And: { + Prefix: 'foo', + Tags: [ + { Key: 'foo', Value: 'bar' }, + { Key: 'foo2', Value: 'bar2' }, + ], + }, + }, }); await s3.send(new PutBucketLifecycleConfigurationCommand(params)); }); @@ -423,12 +434,14 @@ describe('aws-sdk test put bucket lifecycle', () => { return { Bucket: bucket, LifecycleConfiguration: { - Rules: [{ - ID: 'test', - Status: 'Enabled', - Prefix: '', - noncurrentVersionTransition, - }], + Rules: [ + { + ID: 'test', + Status: 'Enabled', + Prefix: '', + noncurrentVersionTransition, + }, + ], }, }; } @@ -464,9 +477,10 @@ describe('aws-sdk test put bucket lifecycle', () => { throw new Error('Expected InvalidArgument error'); } catch (err) { assert.strictEqual(err.name, 'InvalidArgument'); - assert.strictEqual(err.message, - "'NoncurrentDays' in NoncurrentVersionExpiration " + - 'action must be nonnegative'); + assert.strictEqual( + err.message, + "'NoncurrentDays' in NoncurrentVersionExpiration " + 'action must be nonnegative', + ); } }); @@ -487,21 +501,25 @@ describe('aws-sdk test put bucket lifecycle', () => { return { Bucket: bucket, LifecycleConfiguration: { - Rules: [{ - ID: 'test', - Status: 'Enabled', - Prefix: '', - NoncurrentVersionTransitions: noncurrentVersionTransitions, - }], + Rules: [ + { + ID: 'test', + Status: 'Enabled', + Prefix: '', + NoncurrentVersionTransitions: noncurrentVersionTransitions, + }, + ], }, }; } it('should allow config', async () => { - const noncurrentVersionTransitions = [{ - NoncurrentDays: 1, - StorageClass: 'us-east-2', - }]; + const noncurrentVersionTransitions = [ + { + NoncurrentDays: 1, + StorageClass: 'us-east-2', + }, + ]; const params = getParams(noncurrentVersionTransitions); try { await s3.send(new PutBucketLifecycleConfigurationCommand(params)); @@ -514,13 +532,16 @@ describe('aws-sdk test put bucket lifecycle', () => { }); it.skip('should not allow duplicate StorageClass', async () => { - const noncurrentVersionTransitions = [{ - NoncurrentDays: 1, - StorageClass: 'us-east-2', - }, { - NoncurrentDays: 2, - StorageClass: 'us-east-2', - }]; + const noncurrentVersionTransitions = [ + { + NoncurrentDays: 1, + StorageClass: 'us-east-2', + }, + { + NoncurrentDays: 2, + StorageClass: 'us-east-2', + }, + ]; const params = getParams(noncurrentVersionTransitions); try { await s3.send(new PutBucketLifecycleConfigurationCommand(params)); @@ -532,88 +553,111 @@ describe('aws-sdk test put bucket lifecycle', () => { return; } assert.strictEqual(err.name, 'InvalidRequest'); - assert.strictEqual(err.message, + assert.strictEqual( + err.message, "'StorageClass' must be different for " + - "'NoncurrentVersionTransition' actions in same " + - "'Rule' with prefix ''"); + "'NoncurrentVersionTransition' actions in same " + + "'Rule' with prefix ''", + ); } }); it('should not allow unknown StorageClass', async () => { - const noncurrentVersionTransitions = [{ - NoncurrentDays: 1, - StorageClass: 'unknown', - }]; + const noncurrentVersionTransitions = [ + { + NoncurrentDays: 1, + StorageClass: 'unknown', + }, + ]; const params = getParams(noncurrentVersionTransitions); try { await s3.send(new PutBucketLifecycleConfigurationCommand(params)); throw new Error('Expected MalformedXML error'); } catch (err) { - assert(err.name === 'MalformedXML' || err.name === 'NotImplemented', - `Expected MalformedXML or NotImplemented, got ${err.name}`); + assert( + err.name === 'MalformedXML' || err.name === 'NotImplemented', + `Expected MalformedXML or NotImplemented, got ${err.name}`, + ); } }); it(`should not allow NoncurrentDays value exceeding ${MAX_DAYS}`, async () => { - const noncurrentVersionTransitions = [{ - NoncurrentDays: MAX_DAYS + 1, - StorageClass: 'us-east-2', - }]; + const noncurrentVersionTransitions = [ + { + NoncurrentDays: MAX_DAYS + 1, + StorageClass: 'us-east-2', + }, + ]; const params = getParams(noncurrentVersionTransitions); try { await s3.send(new PutBucketLifecycleConfigurationCommand(params)); throw new Error('Expected MalformedXML error'); } catch (err) { - assert(err.name === 'MalformedXML' || err.name === 'NotImplemented', - `Expected MalformedXML or NotImplemented, got ${err.name}`); + assert( + err.name === 'MalformedXML' || err.name === 'NotImplemented', + `Expected MalformedXML or NotImplemented, got ${err.name}`, + ); } }); it('should not allow negative NoncurrentDays', async () => { - const noncurrentVersionTransitions = [{ - NoncurrentDays: -1, - StorageClass: 'us-east-2', - }]; + const noncurrentVersionTransitions = [ + { + NoncurrentDays: -1, + StorageClass: 'us-east-2', + }, + ]; const params = getParams(noncurrentVersionTransitions); try { await s3.send(new PutBucketLifecycleConfigurationCommand(params)); throw new Error('Expected error'); } catch (err) { - assert(err.name === 'InvalidArgument' || err.name === 'NotImplemented', - `Expected InvalidArgument or NotImplemented, got ${err.name}`); + assert( + err.name === 'InvalidArgument' || err.name === 'NotImplemented', + `Expected InvalidArgument or NotImplemented, got ${err.name}`, + ); if (err.name === 'InvalidArgument') { - assert.strictEqual(err.message, - "'NoncurrentDays' in NoncurrentVersionTransition " + - 'action must be nonnegative'); + assert.strictEqual( + err.message, + "'NoncurrentDays' in NoncurrentVersionTransition " + 'action must be nonnegative', + ); } } }); it('should not allow config missing NoncurrentDays', async () => { - const noncurrentVersionTransitions = [{ - StorageClass: 'us-east-2', - }]; + const noncurrentVersionTransitions = [ + { + StorageClass: 'us-east-2', + }, + ]; const params = getParams(noncurrentVersionTransitions); try { await s3.send(new PutBucketLifecycleConfigurationCommand(params)); throw new Error('Expected error'); } catch (err) { - assert(err.name === 'MalformedXML' || err.name === 'NotImplemented', - `Expected MalformedXML or NotImplemented, got ${err.name}`); + assert( + err.name === 'MalformedXML' || err.name === 'NotImplemented', + `Expected MalformedXML or NotImplemented, got ${err.name}`, + ); } }); it('should not allow config missing StorageClass', async () => { - const noncurrentVersionTransitions = [{ - NoncurrentDays: 1, - }]; + const noncurrentVersionTransitions = [ + { + NoncurrentDays: 1, + }, + ]; const params = getParams(noncurrentVersionTransitions); try { await s3.send(new PutBucketLifecycleConfigurationCommand(params)); throw new Error('Expected error'); } catch (err) { - assert(err.name === 'MalformedXML' || err.name === 'NotImplemented', - `Expected MalformedXML or NotImplemented, got ${err.name}`); + assert( + err.name === 'MalformedXML' || err.name === 'NotImplemented', + `Expected MalformedXML or NotImplemented, got ${err.name}`, + ); } }); }); @@ -626,15 +670,19 @@ describe('aws-sdk test put bucket lifecycle', () => { const params = { Bucket: bucket, LifecycleConfiguration: { - Rules: [{ - ID: 'test', - Status: 'Enabled', - Prefix: '', - Transitions: [{ - Days: 2, - StorageClass: 'us-east-2', - }], - }], + Rules: [ + { + ID: 'test', + Status: 'Enabled', + Prefix: '', + Transitions: [ + { + Days: 2, + StorageClass: 'us-east-2', + }, + ], + }, + ], }, }; try { @@ -652,60 +700,72 @@ describe('aws-sdk test put bucket lifecycle', () => { return { Bucket: bucket, LifecycleConfiguration: { - Rules: [{ - ID: 'test', - Status: 'Enabled', - Prefix: '', - Transitions: transitions, - }], + Rules: [ + { + ID: 'test', + Status: 'Enabled', + Prefix: '', + Transitions: transitions, + }, + ], }, }; } it('should allow config', async () => { - const transitions = [{ - Days: 1, - StorageClass: 'us-east-2', - }]; + const transitions = [ + { + Days: 1, + StorageClass: 'us-east-2', + }, + ]; const params = getParams(transitions); await s3.send(new PutBucketLifecycleConfigurationCommand(params)); }); it('should not allow duplicate StorageClass', async () => { - const transitions = [{ - Days: 1, - StorageClass: 'us-east-2', - }, { - Days: 2, - StorageClass: 'us-east-2', - }]; + const transitions = [ + { + Days: 1, + StorageClass: 'us-east-2', + }, + { + Days: 2, + StorageClass: 'us-east-2', + }, + ]; const params = getParams(transitions); try { await s3.send(new PutBucketLifecycleConfigurationCommand(params)); throw new Error('Expected InvalidRequest error'); } catch (err) { assert.strictEqual(err.name, 'InvalidRequest'); - assert.strictEqual(err.message, - "'StorageClass' must be different for 'Transition' " + - "actions in same 'Rule' with prefix ''"); + assert.strictEqual( + err.message, + "'StorageClass' must be different for 'Transition' " + "actions in same 'Rule' with prefix ''", + ); } }); it('should allow Date', async () => { - const transitions = [{ - Date: new Date('2016-01-01T00:00:00.000Z'), - StorageClass: 'us-east-2', - }]; + const transitions = [ + { + Date: new Date('2016-01-01T00:00:00.000Z'), + StorageClass: 'us-east-2', + }, + ]; const params = getParams(transitions); await s3.send(new PutBucketLifecycleConfigurationCommand(params)); }); it('should not allow speficying both Days and Date value', async () => { - const transitions = [{ - Date: new Date('2016-01-01T00:00:00.000Z'), - Days: 1, - StorageClass: 'us-east-2', - }]; + const transitions = [ + { + Date: new Date('2016-01-01T00:00:00.000Z'), + Days: 1, + StorageClass: 'us-east-2', + }, + ]; const params = getParams(transitions); try { await s3.send(new PutBucketLifecycleConfigurationCommand(params)); @@ -716,87 +776,104 @@ describe('aws-sdk test put bucket lifecycle', () => { }); // TODO: Upgrade to aws-sdk >= 2.60.0 for correct Date field support - it.skip('should not allow speficying both Days and Date value ' + - 'across transitions', done => { - const transitions = [{ - Date: '2016-01-01T00:00:00.000Z', - StorageClass: 'us-east-2', - }, { - Days: 1, - StorageClass: 'zenko', - }]; + it.skip('should not allow speficying both Days and Date value ' + 'across transitions', done => { + const transitions = [ + { + Date: '2016-01-01T00:00:00.000Z', + StorageClass: 'us-east-2', + }, + { + Days: 1, + StorageClass: 'zenko', + }, + ]; const params = getParams(transitions); s3.putBucketLifecycleConfiguration(params, err => { assert.strictEqual(err.code, 'InvalidRequest'); - assert.strictEqual(err.message, - "Found mixed 'Date' and 'Days' based Transition " + - "actions in lifecycle rule for prefix ''"); + assert.strictEqual( + err.message, + "Found mixed 'Date' and 'Days' based Transition " + "actions in lifecycle rule for prefix ''", + ); done(); }); }); - it('should not allow speficying both Days and Date value ' + - 'across transitions and expiration', async () => { - const transitions = [{ - Days: 1, - StorageClass: 'us-east-2', - }]; - const params = getParams(transitions); - params.LifecycleConfiguration.Rules[0].Expiration = { - Date: new Date('2016-01-01T00:00:00.000Z') // Use proper Date object - }; - try { - await s3.send(new PutBucketLifecycleConfigurationCommand(params)); - throw new Error('Expected InvalidRequest error'); - } catch (err) { - assert.strictEqual(err.name, 'InvalidRequest'); - assert.strictEqual(err.message, - "Found mixed 'Date' and 'Days' based Expiration and " + - "Transition actions in lifecycle rule for prefix ''"); - } - }); + it( + 'should not allow speficying both Days and Date value ' + 'across transitions and expiration', + async () => { + const transitions = [ + { + Days: 1, + StorageClass: 'us-east-2', + }, + ]; + const params = getParams(transitions); + params.LifecycleConfiguration.Rules[0].Expiration = { + Date: new Date('2016-01-01T00:00:00.000Z'), // Use proper Date object + }; + try { + await s3.send(new PutBucketLifecycleConfigurationCommand(params)); + throw new Error('Expected InvalidRequest error'); + } catch (err) { + assert.strictEqual(err.name, 'InvalidRequest'); + assert.strictEqual( + err.message, + "Found mixed 'Date' and 'Days' based Expiration and " + + "Transition actions in lifecycle rule for prefix ''", + ); + } + }, + ); }); // NoncurrentVersionTransitions not implemented - describe.skip('with NoncurrentVersionTransitions and Transitions', - () => { + describe.skip('with NoncurrentVersionTransitions and Transitions', () => { it('should allow config', async () => { const params = { Bucket: bucket, LifecycleConfiguration: { - Rules: [{ - ID: 'test', - Status: 'Enabled', - Prefix: '', - NoncurrentVersionTransitions: [{ - NoncurrentDays: 1, - StorageClass: 'us-east-2', - }], - Transitions: [{ - Days: 1, - StorageClass: 'us-east-2', - }], - }], + Rules: [ + { + ID: 'test', + Status: 'Enabled', + Prefix: '', + NoncurrentVersionTransitions: [ + { + NoncurrentDays: 1, + StorageClass: 'us-east-2', + }, + ], + Transitions: [ + { + Days: 1, + StorageClass: 'us-east-2', + }, + ], + }, + ], }, }; await s3.send(new PutBucketLifecycleConfigurationCommand(params)); }); }); - it.skip('should not allow config when specifying ' + - 'NoncurrentVersionTransitions', async () => { + it.skip('should not allow config when specifying ' + 'NoncurrentVersionTransitions', async () => { const params = { Bucket: bucket, LifecycleConfiguration: { - Rules: [{ - ID: 'test', - Status: 'Enabled', - Prefix: '', - NoncurrentVersionTransitions: [{ - NoncurrentDays: 1, - StorageClass: 'us-east-2', - }], - }], + Rules: [ + { + ID: 'test', + Status: 'Enabled', + Prefix: '', + NoncurrentVersionTransitions: [ + { + NoncurrentDays: 1, + StorageClass: 'us-east-2', + }, + ], + }, + ], }, }; try { diff --git a/tests/functional/aws-node-sdk/test/bucket/putBucketLogging.js b/tests/functional/aws-node-sdk/test/bucket/putBucketLogging.js index 5bbc81af27..f6f3a5e973 100644 --- a/tests/functional/aws-node-sdk/test/bucket/putBucketLogging.js +++ b/tests/functional/aws-node-sdk/test/bucket/putBucketLogging.js @@ -1,9 +1,5 @@ const assert = require('assert'); -const { - CreateBucketCommand, - PutBucketLoggingCommand, - GetBucketLoggingCommand, -} = require('@aws-sdk/client-s3'); +const { CreateBucketCommand, PutBucketLoggingCommand, GetBucketLoggingCommand } = require('@aws-sdk/client-s3'); const withV4 = require('../support/withV4'); const BucketUtility = require('../../lib/utility/bucket-util'); @@ -55,7 +51,9 @@ function cleanUp(bucketUtil, cb) { throw err; } }), - ]).then(() => cb()).catch(err => cb(err)); + ]) + .then(() => cb()) + .catch(err => cb(err)); } describe('PUT bucket logging', () => { @@ -67,10 +65,12 @@ describe('PUT bucket logging', () => { async function _testPutBucketLoggingError(account, config, statusCode, errMsg, cb) { try { - await account.send(new PutBucketLoggingCommand({ - Bucket: bucketName, - BucketLoggingStatus: config, - })); + await account.send( + new PutBucketLoggingCommand({ + Bucket: bucketName, + BucketLoggingStatus: config, + }), + ); return cb(new Error('Expected error but found none')); } catch (err) { assert(err, 'Expected err but found none'); @@ -100,10 +100,12 @@ describe('PUT bucket logging', () => { afterEach(async () => { process.stdout.write('Deleting buckets\n'); try { - await s3.send(new PutBucketLoggingCommand({ - Bucket: bucketName, - BucketLoggingStatus: {}, - })); + await s3.send( + new PutBucketLoggingCommand({ + Bucket: bucketName, + BucketLoggingStatus: {}, + }), + ); } catch (err) { if (err.name !== 'NoSuchBucket' && err.code !== 'NoSuchBucket') { throw err; @@ -131,17 +133,17 @@ describe('PUT bucket logging', () => { }); it('should put bucket logging configuration successfully', async () => { - await s3.send(new PutBucketLoggingCommand({ - Bucket: bucketName, - BucketLoggingStatus: validLoggingConfig, - })); + await s3.send( + new PutBucketLoggingCommand({ + Bucket: bucketName, + BucketLoggingStatus: validLoggingConfig, + }), + ); const data = await s3.send(new GetBucketLoggingCommand({ Bucket: bucketName })); assert(data.LoggingEnabled); - assert.strictEqual(data.LoggingEnabled.TargetBucket, - targetBucket); - assert.strictEqual(data.LoggingEnabled.TargetPrefix, - 'logs/'); + assert.strictEqual(data.LoggingEnabled.TargetBucket, targetBucket); + assert.strictEqual(data.LoggingEnabled.TargetPrefix, 'logs/'); }); itSkipIfAWS('should return NotImplemented if TargetGrants is present', done => { @@ -149,18 +151,22 @@ describe('PUT bucket logging', () => { }); it('should disable logging with empty BucketLoggingStatus', async () => { - await s3.send(new PutBucketLoggingCommand({ - Bucket: bucketName, - BucketLoggingStatus: validLoggingConfig, - })); + await s3.send( + new PutBucketLoggingCommand({ + Bucket: bucketName, + BucketLoggingStatus: validLoggingConfig, + }), + ); const enabled = await s3.send(new GetBucketLoggingCommand({ Bucket: bucketName })); assert(enabled.LoggingEnabled); - await s3.send(new PutBucketLoggingCommand({ - Bucket: bucketName, - BucketLoggingStatus: {}, - })); + await s3.send( + new PutBucketLoggingCommand({ + Bucket: bucketName, + BucketLoggingStatus: {}, + }), + ); const disabled = await s3.send(new GetBucketLoggingCommand({ Bucket: bucketName })); assert(disabled); @@ -171,22 +177,23 @@ describe('PUT bucket logging', () => { _testPutBucketLoggingError(otherAccountS3, validLoggingConfig, 405, 'MethodNotAllowed', done); }); - it('should return InvalidTargetBucketForLogging if target bucket does not exist', - done => { - const invalidConfig = { - LoggingEnabled: { - TargetBucket: 'nonexistentbucket', - TargetPrefix: 'logs/', - }, - }; - _testPutBucketLoggingError(s3, invalidConfig, 400, 'InvalidTargetBucketForLogging', done); - }); + it('should return InvalidTargetBucketForLogging if target bucket does not exist', done => { + const invalidConfig = { + LoggingEnabled: { + TargetBucket: 'nonexistentbucket', + TargetPrefix: 'logs/', + }, + }; + _testPutBucketLoggingError(s3, invalidConfig, 400, 'InvalidTargetBucketForLogging', done); + }); it('should allow logging when target bucket is owned by same account', async () => { - await s3.send(new PutBucketLoggingCommand({ - Bucket: bucketName, - BucketLoggingStatus: validLoggingConfig, - })); + await s3.send( + new PutBucketLoggingCommand({ + Bucket: bucketName, + BucketLoggingStatus: validLoggingConfig, + }), + ); const data = await s3.send(new GetBucketLoggingCommand({ Bucket: bucketName })); assert(data.LoggingEnabled); @@ -229,8 +236,7 @@ describe('PUT bucket logging', () => { } }); - it('should return InvalidTargetBucketForLogging when target bucket is owned by different account', - async () => { + it('should return InvalidTargetBucketForLogging when target bucket is owned by different account', async () => { const crossAccountConfig = { LoggingEnabled: { TargetBucket: otherAccountTargetBucket, @@ -239,10 +245,12 @@ describe('PUT bucket logging', () => { }; try { - await s3.send(new PutBucketLoggingCommand({ - Bucket: bucketName, - BucketLoggingStatus: crossAccountConfig, - })); + await s3.send( + new PutBucketLoggingCommand({ + Bucket: bucketName, + BucketLoggingStatus: crossAccountConfig, + }), + ); assert.fail('Expected InvalidTargetBucketForLogging error'); } catch (err) { assert.strictEqual(err.name, 'InvalidTargetBucketForLogging'); diff --git a/tests/functional/aws-node-sdk/test/bucket/putBucketNotification.js b/tests/functional/aws-node-sdk/test/bucket/putBucketNotification.js index e50d5c9c1b..5b8faee730 100644 --- a/tests/functional/aws-node-sdk/test/bucket/putBucketNotification.js +++ b/tests/functional/aws-node-sdk/test/bucket/putBucketNotification.js @@ -1,7 +1,9 @@ -const { S3Client, +const { + S3Client, CreateBucketCommand, DeleteBucketCommand, - PutBucketNotificationConfigurationCommand } = require('@aws-sdk/client-s3'); + PutBucketNotificationConfigurationCommand, +} = require('@aws-sdk/client-s3'); const checkError = require('../../lib/utility/checkError'); const getConfig = require('../support/config'); @@ -52,7 +54,7 @@ describe('aws-sdk test put notification configuration', () => { }); describe('config rules', () => { - beforeEach(() => s3.send(new CreateBucketCommand({Bucket: bucket}))); + beforeEach(() => s3.send(new CreateBucketCommand({ Bucket: bucket }))); afterEach(() => s3.send(new DeleteBucketCommand({ Bucket: bucket }))); @@ -72,8 +74,7 @@ describe('aws-sdk test put notification configuration', () => { }); it('should put notification configuration on bucket with multiple events', async () => { - const params = getNotificationParams( - ['s3:ObjectCreated:*', 's3:ObjectRemoved:*']); + const params = getNotificationParams(['s3:ObjectCreated:*', 's3:ObjectRemoved:*']); await s3.send(new PutBucketNotificationConfigurationCommand(params)); }); @@ -136,29 +137,35 @@ describe('aws-sdk test put notification configuration', () => { it(`should handle ${event} events based on lifecycle rules configuration`, done => { const params = getNotificationParams([event]); const shouldSucceed = config.supportedLifecycleRules.some(rule => rule.includes(supported)); - s3.send(new PutBucketNotificationConfigurationCommand(params)).then(() => { - if (shouldSucceed) { - done(); - } else { - done(new Error('Expected MalformedXML error but operation succeeded')); - } - }).catch(err => { - if (shouldSucceed) { - done(err); - } else { - checkError(err, 'MalformedXML', 400); - done(); - } - }); + s3.send(new PutBucketNotificationConfigurationCommand(params)) + .then(() => { + if (shouldSucceed) { + done(); + } else { + done(new Error('Expected MalformedXML error but operation succeeded')); + } + }) + .catch(err => { + if (shouldSucceed) { + done(err); + } else { + checkError(err, 'MalformedXML', 400); + done(); + } + }); }); }); }); }); describe('cross origin requests', () => { - beforeEach(() => s3.send(new CreateBucketCommand({ - Bucket: bucket, - }))); + beforeEach(() => + s3.send( + new CreateBucketCommand({ + Bucket: bucket, + }), + ), + ); afterEach(() => s3.send(new DeleteBucketCommand({ Bucket: bucket }))); @@ -167,11 +174,13 @@ describe('aws-sdk test put notification configuration', () => { it: 'return valid error with invalid arn', param: getNotificationParams(null, 'invalidArn'), error: 'MalformedXML', - }, { + }, + { it: 'return valid error with unknown/unsupported destination', param: getNotificationParams(null, 'arn:scality:bucketnotif:::target100'), error: 'InvalidArgument', - }, { + }, + { it: 'save notification configuration with correct arn', param: getNotificationParams(), }, diff --git a/tests/functional/aws-node-sdk/test/bucket/putBucketObjectLock.js b/tests/functional/aws-node-sdk/test/bucket/putBucketObjectLock.js index 460c42d87e..94757c3d48 100644 --- a/tests/functional/aws-node-sdk/test/bucket/putBucketObjectLock.js +++ b/tests/functional/aws-node-sdk/test/bucket/putBucketObjectLock.js @@ -1,7 +1,9 @@ -const { S3Client, +const { + S3Client, CreateBucketCommand, DeleteBucketCommand, - PutObjectLockConfigurationCommand } = require('@aws-sdk/client-s3'); + PutObjectLockConfigurationCommand, +} = require('@aws-sdk/client-s3'); const checkError = require('../../lib/utility/checkError'); const getConfig = require('../support/config'); @@ -51,7 +53,7 @@ describe('aws-sdk test put object lock configuration', () => { }); describe('on object lock disabled bucket', () => { - beforeEach(() => s3.send(new CreateBucketCommand({Bucket: bucket}))); + beforeEach(() => s3.send(new CreateBucketCommand({ Bucket: bucket }))); afterEach(() => s3.send(new DeleteBucketCommand({ Bucket: bucket }))); @@ -82,10 +84,14 @@ describe('aws-sdk test put object lock configuration', () => { }); describe('config rules', () => { - beforeEach(() => s3.send(new CreateBucketCommand({ - Bucket: bucket, - ObjectLockEnabledForBucket: true, - }))); + beforeEach(() => + s3.send( + new CreateBucketCommand({ + Bucket: bucket, + ObjectLockEnabledForBucket: true, + }), + ), + ); afterEach(() => s3.send(new DeleteBucketCommand({ Bucket: bucket }))); diff --git a/tests/functional/aws-node-sdk/test/bucket/putBucketPolicy.js b/tests/functional/aws-node-sdk/test/bucket/putBucketPolicy.js index d2c12da27c..37ff5b31b0 100644 --- a/tests/functional/aws-node-sdk/test/bucket/putBucketPolicy.js +++ b/tests/functional/aws-node-sdk/test/bucket/putBucketPolicy.js @@ -1,9 +1,6 @@ const assert = require('assert'); const { errors } = require('arsenal'); -const { S3Client, - CreateBucketCommand, - DeleteBucketCommand, - PutBucketPolicyCommand } = require('@aws-sdk/client-s3'); +const { S3Client, CreateBucketCommand, DeleteBucketCommand, PutBucketPolicyCommand } = require('@aws-sdk/client-s3'); const getConfig = require('../support/config'); const BucketUtility = require('../../lib/utility/bucket-util'); @@ -55,9 +52,7 @@ function generateRandomString(length) { const allowedCharacters = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789+=,.@ -/'; const allowedCharactersLength = allowedCharacters.length; - return [...Array(length)] - .map(() => allowedCharacters[~~(Math.random() * allowedCharactersLength)]) - .join(''); + return [...Array(length)].map(() => allowedCharacters[~~(Math.random() * allowedCharactersLength)]).join(''); } // Check for the expected error response code and status code. @@ -65,15 +60,20 @@ function assertError(err, expectedErr) { if (expectedErr === null) { assert.strictEqual(err, null, `expected no error but got '${err}'`); } else { - assert.strictEqual(err.name, expectedErr, 'incorrect error response ' + - `code: should be '${expectedErr}' but got '${err.name}'`); - assert.strictEqual(err.$metadata.httpStatusCode, errors[expectedErr].code, + assert.strictEqual( + err.name, + expectedErr, + 'incorrect error response ' + `code: should be '${expectedErr}' but got '${err.name}'`, + ); + assert.strictEqual( + err.$metadata.httpStatusCode, + errors[expectedErr].code, 'incorrect error status code: should be ' + - `${errors[expectedErr].code}, but got '${err.$metadata.httpStatusCode}'`); + `${errors[expectedErr].code}, but got '${err.$metadata.httpStatusCode}'`, + ); } } - describe('aws-sdk test put bucket policy', () => { let s3; let otherAccountS3; @@ -170,12 +170,18 @@ describe('aws-sdk test put bucket policy', () => { }); it('should allow bucket policy with pincipal arn less than 2048 characters', async () => { - const params = getPolicyParams({ key: 'Principal', value: { AWS: `arn:aws:iam::767707094035:user/${generateRandomString(150)}` } }); + const params = getPolicyParams({ + key: 'Principal', + value: { AWS: `arn:aws:iam::767707094035:user/${generateRandomString(150)}` }, + }); await s3.send(new PutBucketPolicyCommand(params)); }); it('should not allow bucket policy with pincipal arn more than 2048 characters', async () => { - const params = getPolicyParams({ key: 'Principal', value: { AWS: `arn:aws:iam::767707094035:user/${generateRandomString(2020)}` } }); + const params = getPolicyParams({ + key: 'Principal', + value: { AWS: `arn:aws:iam::767707094035:user/${generateRandomString(2020)}` }, + }); try { await s3.send(new PutBucketPolicyCommand(params)); throw new Error('Expected MalformedPolicy error'); @@ -186,7 +192,8 @@ describe('aws-sdk test put bucket policy', () => { it('should allow bucket policy with valid SourceIp condition', async () => { const params = getPolicyParams({ - key: 'Condition', value: { + key: 'Condition', + value: { IpAddress: { 'aws:SourceIp': '192.168.100.0/24', }, @@ -197,7 +204,8 @@ describe('aws-sdk test put bucket policy', () => { it('should not allow bucket policy with invalid SourceIp format', async () => { const params = getPolicyParams({ - key: 'Condition', value: { + key: 'Condition', + value: { IpAddress: { 'aws:SourceIp': '192.168.100', // Invalid IP format }, @@ -213,7 +221,8 @@ describe('aws-sdk test put bucket policy', () => { it('should allow bucket policy with valid s3:object-lock-remaining-retention-days condition', async () => { const params = getPolicyParams({ - key: 'Condition', value: { + key: 'Condition', + value: { NumericGreaterThanEquals: { 's3:object-lock-remaining-retention-days': '30', }, @@ -225,7 +234,8 @@ describe('aws-sdk test put bucket policy', () => { // yep, this is the expected behaviour it('should not reject policy with invalid s3:object-lock-remaining-retention-days value', async () => { const params = getPolicyParams({ - key: 'Condition', value: { + key: 'Condition', + value: { NumericGreaterThanEquals: { 's3:object-lock-remaining-retention-days': '-1', // Invalid value }, @@ -237,7 +247,8 @@ describe('aws-sdk test put bucket policy', () => { // this too ¯\_(ツ)_/¯ it('should not reject policy with a key starting with aws:', async () => { const params = getPolicyParams({ - key: 'Condition', value: { + key: 'Condition', + value: { NumericGreaterThanEquals: { 'aws:have-a-nice-day': 'blabla', // Invalid value }, @@ -248,7 +259,8 @@ describe('aws-sdk test put bucket policy', () => { it('should reject policy with a key that does not exist that does not start with aws:', async () => { const params = getPolicyParams({ - key: 'Condition', value: { + key: 'Condition', + value: { NumericGreaterThanEquals: { 'have-a-nice-day': 'blabla', // Invalid value }, @@ -264,7 +276,8 @@ describe('aws-sdk test put bucket policy', () => { it('should enforce policies with both SourceIp and s3:object-lock conditions together', async () => { const params = getPolicyParams({ - key: 'Condition', value: { + key: 'Condition', + value: { IpAddress: { 'aws:SourceIp': '192.168.100.0/24', }, @@ -278,7 +291,8 @@ describe('aws-sdk test put bucket policy', () => { it('should return error if a condition one of the condition values is invalid', async () => { const params = getPolicyParams({ - key: 'Condition', value: { + key: 'Condition', + value: { IpAddress: { 'aws:SourceIp': '192.168.100', }, diff --git a/tests/functional/aws-node-sdk/test/bucket/putBucketRateLimit.js b/tests/functional/aws-node-sdk/test/bucket/putBucketRateLimit.js index c2d0a94ba6..48b5099e49 100644 --- a/tests/functional/aws-node-sdk/test/bucket/putBucketRateLimit.js +++ b/tests/functional/aws-node-sdk/test/bucket/putBucketRateLimit.js @@ -1,9 +1,5 @@ const assert = require('assert'); -const { - S3Client, - CreateBucketCommand, - DeleteBucketCommand, -} = require('@aws-sdk/client-s3'); +const { S3Client, CreateBucketCommand, DeleteBucketCommand } = require('@aws-sdk/client-s3'); const getConfig = require('../support/config'); const { sendRateLimitRequest, skipIfRateLimitDisabled } = require('../rateLimit/tooling'); const { config } = require('../../../../../lib/Config'); @@ -39,8 +35,12 @@ skipIfRateLimitDisabled('Test put bucket rate limit', () => { it('should set the rate limit config', async () => { try { - await sendRateLimitRequest('PUT', '127.0.0.1:8000', - `/${bucket}/?rate-limit`, JSON.stringify(rateLimitConfig)); + await sendRateLimitRequest( + 'PUT', + '127.0.0.1:8000', + `/${bucket}/?rate-limit`, + JSON.stringify(rateLimitConfig), + ); assert.ok(true); } catch (err) { assert.ifError(err); @@ -50,15 +50,22 @@ skipIfRateLimitDisabled('Test put bucket rate limit', () => { it('should update existing rate limit config', async () => { try { const initialConfig = { RequestsPerSecond: 100 }; - await sendRateLimitRequest('PUT', '127.0.0.1:8000', - `/${bucket}/?rate-limit`, JSON.stringify(initialConfig)); - - await sendRateLimitRequest('PUT', '127.0.0.1:8000', - `/${bucket}/?rate-limit`, JSON.stringify(rateLimitConfig)); + await sendRateLimitRequest( + 'PUT', + '127.0.0.1:8000', + `/${bucket}/?rate-limit`, + JSON.stringify(initialConfig), + ); + + await sendRateLimitRequest( + 'PUT', + '127.0.0.1:8000', + `/${bucket}/?rate-limit`, + JSON.stringify(rateLimitConfig), + ); // Verify the update - const data = await sendRateLimitRequest('GET', '127.0.0.1:8000', - `/${bucket}/?rate-limit`); + const data = await sendRateLimitRequest('GET', '127.0.0.1:8000', `/${bucket}/?rate-limit`); assert.strictEqual(data.RequestsPerSecond.Limit, 200); } catch (err) { assert.ifError(err); @@ -67,8 +74,12 @@ skipIfRateLimitDisabled('Test put bucket rate limit', () => { it('should return NoSuchBucket error when bucket does not exist', async () => { try { - await sendRateLimitRequest('PUT', '127.0.0.1:8000', - `/${nonExistentBucket}/?rate-limit`, JSON.stringify(rateLimitConfig)); + await sendRateLimitRequest( + 'PUT', + '127.0.0.1:8000', + `/${nonExistentBucket}/?rate-limit`, + JSON.stringify(rateLimitConfig), + ); } catch (err) { assert.strictEqual(err.Error.Code[0], 'NoSuchBucket'); } @@ -76,8 +87,12 @@ skipIfRateLimitDisabled('Test put bucket rate limit', () => { it('should return InvalidArgument error when RequestsPerSecond is negative', async () => { try { - await sendRateLimitRequest('PUT', '127.0.0.1:8000', - `/${bucket}/?rate-limit`, JSON.stringify(invalidConfig)); + await sendRateLimitRequest( + 'PUT', + '127.0.0.1:8000', + `/${bucket}/?rate-limit`, + JSON.stringify(invalidConfig), + ); } catch (err) { assert.strictEqual(err.Error.Code[0], 'InvalidArgument'); } @@ -85,8 +100,12 @@ skipIfRateLimitDisabled('Test put bucket rate limit', () => { it('should return InvalidArgument error when RequestsPerSecond is not an integer', async () => { try { - await sendRateLimitRequest('PUT', '127.0.0.1:8000', - `/${bucket}/?rate-limit`, JSON.stringify(invalidConfigNotInteger)); + await sendRateLimitRequest( + 'PUT', + '127.0.0.1:8000', + `/${bucket}/?rate-limit`, + JSON.stringify(invalidConfigNotInteger), + ); } catch (err) { assert.strictEqual(err.Error.Code[0], 'InvalidArgument'); } @@ -94,8 +113,12 @@ skipIfRateLimitDisabled('Test put bucket rate limit', () => { it('should return InvalidArgument error when RequestsPerSecond is missing', async () => { try { - await sendRateLimitRequest('PUT', '127.0.0.1:8000', - `/${bucket}/?rate-limit`, JSON.stringify(missingLimitConfig)); + await sendRateLimitRequest( + 'PUT', + '127.0.0.1:8000', + `/${bucket}/?rate-limit`, + JSON.stringify(missingLimitConfig), + ); } catch (err) { assert.strictEqual(err.Error.Code[0], 'InvalidArgument'); } @@ -103,8 +126,7 @@ skipIfRateLimitDisabled('Test put bucket rate limit', () => { it('should return InvalidArgument error when request body is invalid JSON', async () => { try { - await sendRateLimitRequest('PUT', '127.0.0.1:8000', - `/${bucket}/?rate-limit`, 'invalid json{'); + await sendRateLimitRequest('PUT', '127.0.0.1:8000', `/${bucket}/?rate-limit`, 'invalid json{'); } catch (err) { assert.strictEqual(err.Error.Code[0], 'InvalidArgument'); } @@ -113,19 +135,16 @@ skipIfRateLimitDisabled('Test put bucket rate limit', () => { it('should allow zero as a valid RequestsPerSecond value', async () => { try { const zeroConfig = { RequestsPerSecond: 0 }; - await sendRateLimitRequest('PUT', '127.0.0.1:8000', - `/${bucket}/?rate-limit`, JSON.stringify(zeroConfig)); + await sendRateLimitRequest('PUT', '127.0.0.1:8000', `/${bucket}/?rate-limit`, JSON.stringify(zeroConfig)); - const data = await sendRateLimitRequest('GET', '127.0.0.1:8000', - `/${bucket}/?rate-limit`); - assert.deepStrictEqual(data, { RequestsPerSecond: { Limit: 0 } }); + const data = await sendRateLimitRequest('GET', '127.0.0.1:8000', `/${bucket}/?rate-limit`); + assert.deepStrictEqual(data, { RequestsPerSecond: { Limit: 0 } }); } catch (err) { assert.ifError(err); } }); describe('validation against node and worker count', () => { - const nodes = config.rateLimiting?.nodes || 1; const workers = config.clusters || 1; const minLimit = nodes * workers; @@ -138,8 +157,12 @@ skipIfRateLimitDisabled('Test put bucket rate limit', () => { let error; try { const invalidConfig = { RequestsPerSecond: minLimit - 1 }; - await sendRateLimitRequest('PUT', '127.0.0.1:8000', - `/${bucket}/?rate-limit`, JSON.stringify(invalidConfig)); + await sendRateLimitRequest( + 'PUT', + '127.0.0.1:8000', + `/${bucket}/?rate-limit`, + JSON.stringify(invalidConfig), + ); } catch (err) { error = err; } finally { @@ -151,11 +174,14 @@ skipIfRateLimitDisabled('Test put bucket rate limit', () => { it('should accept limits equal to (nodes x workers)', async () => { try { const validConfig = { RequestsPerSecond: minLimit }; - await sendRateLimitRequest('PUT', '127.0.0.1:8000', - `/${bucket}/?rate-limit`, JSON.stringify(validConfig)); - - const data = await sendRateLimitRequest('GET', '127.0.0.1:8000', - `/${bucket}/?rate-limit`); + await sendRateLimitRequest( + 'PUT', + '127.0.0.1:8000', + `/${bucket}/?rate-limit`, + JSON.stringify(validConfig), + ); + + const data = await sendRateLimitRequest('GET', '127.0.0.1:8000', `/${bucket}/?rate-limit`); assert.strictEqual(data.RequestsPerSecond.Limit, minLimit); } catch (err) { assert.ifError(err); @@ -165,11 +191,14 @@ skipIfRateLimitDisabled('Test put bucket rate limit', () => { it('should accept limits greater than (nodes x workers)', async () => { try { const validConfig = { RequestsPerSecond: minLimit + 1000 }; - await sendRateLimitRequest('PUT', '127.0.0.1:8000', - `/${bucket}/?rate-limit`, JSON.stringify(validConfig)); - - const data = await sendRateLimitRequest('GET', '127.0.0.1:8000', - `/${bucket}/?rate-limit`); + await sendRateLimitRequest( + 'PUT', + '127.0.0.1:8000', + `/${bucket}/?rate-limit`, + JSON.stringify(validConfig), + ); + + const data = await sendRateLimitRequest('GET', '127.0.0.1:8000', `/${bucket}/?rate-limit`); assert.strictEqual(data.RequestsPerSecond.Limit, minLimit + 1000); } catch (err) { assert.ifError(err); diff --git a/tests/functional/aws-node-sdk/test/bucket/putBucketReplication.js b/tests/functional/aws-node-sdk/test/bucket/putBucketReplication.js index 4abb996f8e..9ac0dc35b9 100644 --- a/tests/functional/aws-node-sdk/test/bucket/putBucketReplication.js +++ b/tests/functional/aws-node-sdk/test/bucket/putBucketReplication.js @@ -1,12 +1,14 @@ const assert = require('assert'); const { errors } = require('arsenal'); -const { S3Client, +const { + S3Client, CreateBucketCommand, DeleteBucketCommand, DeleteBucketCorsCommand, PutBucketCorsCommand, PutBucketReplicationCommand, - PutBucketVersioningCommand } = require('@aws-sdk/client-s3'); + PutBucketVersioningCommand, +} = require('@aws-sdk/client-s3'); const { series } = require('async'); const getConfig = require('../support/config'); @@ -14,7 +16,6 @@ const replicationUtils = require('../../lib/utility/replication'); const BucketUtility = require('../../lib/utility/bucket-util'); const itSkipIfE2E = process.env.S3_END_TO_END ? it.skip : it; - const sourceBucket = 'source-bucket'; const destinationBucket = 'destination-bucket'; @@ -23,11 +24,17 @@ function assertError(err, expectedErr) { if (expectedErr === null) { assert.strictEqual(err, null, `expected no error but got '${err}'`); } else { - assert.strictEqual(err.name, expectedErr, 'incorrect error response ' + - `code: should be '${expectedErr}' but got '${err.name}'`); - assert.strictEqual(err.$metadata.httpStatusCode, errors[expectedErr].code, + assert.strictEqual( + err.name, + expectedErr, + 'incorrect error response ' + `code: should be '${expectedErr}' but got '${err.name}'`, + ); + assert.strictEqual( + err.$metadata.httpStatusCode, + errors[expectedErr].code, `incorrect error status code: should be ${errors[expectedErr].code} but got ` + - `'${err.$metadata.httpStatusCode}'`); + `'${err.$metadata.httpStatusCode}'`, + ); } } @@ -51,8 +58,7 @@ function getVersioningParams(status) { // Get a complete replication configuration, or remove the specified property. const replicationConfig = { - Role: 'arn:aws:iam::account-id:role/src-resource,' + - 'arn:aws:iam::account-id:role/dest-resource', + Role: 'arn:aws:iam::account-id:role/src-resource,' + 'arn:aws:iam::account-id:role/dest-resource', Rules: [ { Destination: { @@ -69,8 +75,7 @@ const replicationConfig = { // Set the rules array of a configuration or a property of the first rule. function setConfigRules(val) { const config = Object.assign({}, replicationConfig); - config.Rules = Array.isArray(val) ? val : - [Object.assign({}, config.Rules[0], val)]; + config.Rules = Array.isArray(val) ? val : [Object.assign({}, config.Rules[0], val)]; return config; } @@ -82,16 +87,23 @@ describe('aws-node-sdk test putBucketReplication bucket status', () => { function checkVersioningError(s3Client, versioningStatus, expectedErr) { const versioningParams = getVersioningParams(versioningStatus); - return series([ - next => s3Client.send(new PutBucketVersioningCommand(versioningParams)) - .then(() => next()) - .catch(next), - next => s3Client.send(new PutBucketReplicationCommand(replicationParams)) - .then(() => next()) - .catch(next), - ], err => { - assertError(err, expectedErr); - }); + return series( + [ + next => + s3Client + .send(new PutBucketVersioningCommand(versioningParams)) + .then(() => next()) + .catch(next), + next => + s3Client + .send(new PutBucketReplicationCommand(replicationParams)) + .then(() => next()) + .catch(next), + ], + err => { + assertError(err, expectedErr); + }, + ); } before(() => { @@ -101,7 +113,7 @@ describe('aws-node-sdk test putBucketReplication bucket status', () => { replicationAccountS3 = new BucketUtility('replication', {}).s3; }); - it('should return \'NoSuchBucket\' error if bucket does not exist', async () => { + it("should return 'NoSuchBucket' error if bucket does not exist", async () => { try { await s3.send(new PutBucketReplicationCommand(replicationParams)); throw new Error('Expected NoSuchBucket error'); @@ -112,7 +124,7 @@ describe('aws-node-sdk test putBucketReplication bucket status', () => { assertError(err, 'NoSuchBucket'); } }); - + describe('test putBucketReplication bucket versioning status', () => { beforeEach(() => s3.send(new CreateBucketCommand({ Bucket: sourceBucket }))); @@ -130,7 +142,7 @@ describe('aws-node-sdk test putBucketReplication bucket status', () => { assert.strictEqual(err.$metadata.httpStatusCode, 403); } }); - + it('should not put configuration on bucket without versioning', async () => { try { await s3.send(new PutBucketReplicationCommand(replicationParams)); @@ -143,19 +155,18 @@ describe('aws-node-sdk test putBucketReplication bucket status', () => { } }); - it('should not put configuration on bucket with \'Suspended\'' + - 'versioning', () => - checkVersioningError(s3, 'Suspended', 'InvalidRequest')); + it("should not put configuration on bucket with 'Suspended'" + 'versioning', () => + checkVersioningError(s3, 'Suspended', 'InvalidRequest'), + ); - it('should put configuration on a bucket with versioning', () => - checkVersioningError(s3, 'Enabled', null)); + it('should put configuration on a bucket with versioning', () => checkVersioningError(s3, 'Enabled', null)); // S3C doesn't support service account. There is no cross account access for replication account. // (canonicalId looking like http://acs.zenko.io/accounts/service/replication) const itSkipS3C = process.env.S3_END_TO_END ? it.skip : it; - itSkipS3C('should put configuration on a bucket with versioning if ' + - 'user is a replication user', () => - checkVersioningError(replicationAccountS3, 'Enabled', null)); + itSkipS3C('should put configuration on a bucket with versioning if ' + 'user is a replication user', () => + checkVersioningError(replicationAccountS3, 'Enabled', null), + ); }); }); @@ -164,7 +175,8 @@ describe('aws-node-sdk test putBucketReplication configuration rules', () => { function checkError(config, expectedErr) { const replicationParams = getReplicationParams(config); - return s3.send(new PutBucketReplicationCommand(replicationParams)) + return s3 + .send(new PutBucketReplicationCommand(replicationParams)) .then(() => { if (expectedErr !== null) { return Promise.reject(new Error(`Expected ${expectedErr} error`)); @@ -190,21 +202,26 @@ describe('aws-node-sdk test putBucketReplication configuration rules', () => { const Role = ARN === '' || ARN === ',' ? ARN : `${ARN},${ARN}`; const config = Object.assign({}, replicationConfig, { Role }); - it('should not accept configuration when \'Role\' is not a ' + - 'comma-separated list of two valid Amazon Resource Names: ' + - `'${Role}'`, () => - checkError(config, 'InvalidArgument')); + it( + "should not accept configuration when 'Role' is not a " + + 'comma-separated list of two valid Amazon Resource Names: ' + + `'${Role}'`, + () => checkError(config, 'InvalidArgument'), + ); }); - it('should not accept configuration when \'Role\' is a comma-separated ' + - 'list of more than two valid Amazon Resource Names', + it( + "should not accept configuration when 'Role' is a comma-separated " + + 'list of more than two valid Amazon Resource Names', () => { - const Role = 'arn:aws:iam::account-id:role/resource-1,' + + const Role = + 'arn:aws:iam::account-id:role/resource-1,' + 'arn:aws:iam::account-id:role/resource-2,' + 'arn:aws:iam::account-id:role/resource-3'; const config = Object.assign({}, replicationConfig, { Role }); checkError(config, 'InvalidArgument'); - }); + }, + ); replicationUtils.validRoleARNs.forEach(ARN => { const config = setConfigRules({ @@ -219,62 +236,68 @@ describe('aws-node-sdk test putBucketReplication configuration rules', () => { itSkipIfE2E(test, () => checkError(config, null)); }); - it('should allow a combination of storageClasses across rules', () => { - const config = setConfigRules([replicationConfig.Rules[0], { - Destination: { - Bucket: `arn:aws:s3:::${destinationBucket}`, - StorageClass: 'us-east-2', + it('should allow a combination of storageClasses across rules', () => { + const config = setConfigRules([ + replicationConfig.Rules[0], + { + Destination: { + Bucket: `arn:aws:s3:::${destinationBucket}`, + StorageClass: 'us-east-2', + }, + Prefix: 'bar', + Status: 'Enabled', }, - Prefix: 'bar', - Status: 'Enabled', - }]); - config.Role = 'arn:aws:iam::account-id:role/resource,' + - 'arn:aws:iam::account-id:role/resource1'; + ]); + config.Role = 'arn:aws:iam::account-id:role/resource,' + 'arn:aws:iam::account-id:role/resource1'; checkError(config, null); }); - itSkipIfE2E('should not allow a comma separated list of roles when' + - ' a rule storageClass defines an external location', () => { - const config = { - Role: 'arn:aws:iam::account-id:role/src-resource,' + - 'arn:aws:iam::account-id:role/dest-resource', - Rules: [ - { - Destination: { - Bucket: `arn:aws:s3:::${destinationBucket}`, - StorageClass: 'us-east-2', + itSkipIfE2E( + 'should not allow a comma separated list of roles when' + ' a rule storageClass defines an external location', + () => { + const config = { + Role: 'arn:aws:iam::account-id:role/src-resource,' + 'arn:aws:iam::account-id:role/dest-resource', + Rules: [ + { + Destination: { + Bucket: `arn:aws:s3:::${destinationBucket}`, + StorageClass: 'us-east-2', + }, + Prefix: 'test-prefix', + Status: 'Enabled', }, - Prefix: 'test-prefix', - Status: 'Enabled', - }, - ], - }; - checkError(config, 'InvalidArgument'); - }); + ], + }; + checkError(config, 'InvalidArgument'); + }, + ); replicationUtils.validRoleARNs.forEach(ARN => { const Role = `${ARN},${ARN}`; const config = Object.assign({}, replicationConfig, { Role }); - it('should accept configuration when \'Role\' is a comma-separated ' + - `list of two valid Amazon Resource Names: '${Role}'`, () => - checkError(config, null)); + it( + "should accept configuration when 'Role' is a comma-separated " + + `list of two valid Amazon Resource Names: '${Role}'`, + () => checkError(config, null), + ); }); replicationUtils.invalidBucketARNs.forEach(ARN => { const config = setConfigRules({ Destination: { Bucket: ARN } }); - it('should not accept configuration when \'Bucket\' is not a ' + - `valid Amazon Resource Name format: '${ARN}'`, () => - checkError(config, 'InvalidArgument')); + it( + "should not accept configuration when 'Bucket' is not a " + `valid Amazon Resource Name format: '${ARN}'`, + () => checkError(config, 'InvalidArgument'), + ); }); - it('should not accept configuration when \'Rules\' is empty ', () => { + it("should not accept configuration when 'Rules' is empty ", () => { const config = Object.assign({}, replicationConfig, { Rules: [] }); return checkError(config, 'MalformedXML'); }); - it('should not accept configuration when \'Rules\' is > 1000', () => { + it("should not accept configuration when 'Rules' is > 1000", () => { const arr = []; for (let i = 0; i < 1001; i++) { arr.push({ @@ -287,13 +310,13 @@ describe('aws-node-sdk test putBucketReplication configuration rules', () => { return checkError(config, 'InvalidRequest'); }); - it('should not accept configuration when \'ID\' length is > 255', () => { + it("should not accept configuration when 'ID' length is > 255", () => { // Set ID to a string of length 256. const config = setConfigRules({ ID: new Array(257).join('x') }); return checkError(config, 'InvalidArgument'); }); - it('should not accept configuration when \'ID\' is not unique', () => { + it("should not accept configuration when 'ID' is not unique", () => { const rule1 = replicationConfig.Rules[0]; // Prefix is unique, but not the ID. const rule2 = Object.assign({}, rule1, { Prefix: 'bar' }); @@ -301,8 +324,7 @@ describe('aws-node-sdk test putBucketReplication configuration rules', () => { return checkError(config, 'InvalidRequest'); }); - it('should accept configuration when \'ID\' is not provided for multiple ' + - 'rules', () => { + it("should accept configuration when 'ID' is not provided for multiple " + 'rules', () => { const replicationConfigWithoutID = Object.assign({}, replicationConfig); const rule1 = replicationConfigWithoutID.Rules[0]; delete rule1.ID; @@ -314,60 +336,75 @@ describe('aws-node-sdk test putBucketReplication configuration rules', () => { replicationUtils.validStatuses.forEach(status => { const config = setConfigRules({ Status: status }); - it(`should accept configuration when 'Role' is ${status}`, () => - checkError(config, null)); + it(`should accept configuration when 'Role' is ${status}`, () => checkError(config, null)); }); - it('should not accept configuration when \'Status\' is invalid', () => { + it("should not accept configuration when 'Status' is invalid", () => { // Status must either be 'Enabled' or 'Disabled'. const config = setConfigRules({ Status: 'Invalid' }); return checkError(config, 'MalformedXML'); }); - it('should accept configuration when \'Prefix\' is \'\'', - () => { - const config = setConfigRules({ Prefix: '' }); - return checkError(config, null); - }); + it("should accept configuration when 'Prefix' is ''", () => { + const config = setConfigRules({ Prefix: '' }); + return checkError(config, null); + }); - it('should not accept configuration when \'Prefix\' length is > 1024', - () => { - // Set Prefix to a string of length of 1025. - const config = setConfigRules({ - Prefix: new Array(1026).join('x'), - }); - return checkError(config, 'InvalidArgument'); + it("should not accept configuration when 'Prefix' length is > 1024", () => { + // Set Prefix to a string of length of 1025. + const config = setConfigRules({ + Prefix: new Array(1026).join('x'), }); - - it('should not accept configuration when rules contain overlapping ' + - '\'Prefix\' values: new prefix starts with used prefix', () => { - const config = setConfigRules([replicationConfig.Rules[0], { - Destination: { Bucket: `arn:aws:s3:::${destinationBucket}` }, - Prefix: 'test-prefix/more-content', - Status: 'Enabled', - }]); - return checkError(config, 'InvalidRequest'); + return checkError(config, 'InvalidArgument'); }); - it('should not accept configuration when rules contain overlapping ' + - '\'Prefix\' values: used prefix starts with new prefix', () => { - const config = setConfigRules([replicationConfig.Rules[0], { - Destination: { Bucket: `arn:aws:s3:::${destinationBucket}` }, - Prefix: 'test', - Status: 'Enabled', - }]); - return checkError(config, 'InvalidRequest'); - }); + it( + 'should not accept configuration when rules contain overlapping ' + + "'Prefix' values: new prefix starts with used prefix", + () => { + const config = setConfigRules([ + replicationConfig.Rules[0], + { + Destination: { Bucket: `arn:aws:s3:::${destinationBucket}` }, + Prefix: 'test-prefix/more-content', + Status: 'Enabled', + }, + ]); + return checkError(config, 'InvalidRequest'); + }, + ); - it('should not accept configuration when \'Destination\' properties of ' + - 'two or more rules specify different buckets', () => { - const config = setConfigRules([replicationConfig.Rules[0], { - Destination: { Bucket: `arn:aws:s3:::${destinationBucket}-1` }, - Prefix: 'bar', - Status: 'Enabled', - }]); - return checkError(config, 'InvalidRequest'); - }); + it( + 'should not accept configuration when rules contain overlapping ' + + "'Prefix' values: used prefix starts with new prefix", + () => { + const config = setConfigRules([ + replicationConfig.Rules[0], + { + Destination: { Bucket: `arn:aws:s3:::${destinationBucket}` }, + Prefix: 'test', + Status: 'Enabled', + }, + ]); + return checkError(config, 'InvalidRequest'); + }, + ); + + it( + "should not accept configuration when 'Destination' properties of " + + 'two or more rules specify different buckets', + () => { + const config = setConfigRules([ + replicationConfig.Rules[0], + { + Destination: { Bucket: `arn:aws:s3:::${destinationBucket}-1` }, + Prefix: 'bar', + Status: 'Enabled', + }, + ]); + return checkError(config, 'InvalidRequest'); + }, + ); replicationUtils.validStorageClasses.forEach(storageClass => { const config = setConfigRules({ @@ -377,8 +414,7 @@ describe('aws-node-sdk test putBucketReplication configuration rules', () => { }, }); - it('should accept configuration when \'StorageClass\' is ' + - `${storageClass}`, () => checkError(config, null)); + it("should accept configuration when 'StorageClass' is " + `${storageClass}`, () => checkError(config, null)); }); // A combination of external destination storage classes. @@ -390,20 +426,20 @@ describe('aws-node-sdk test putBucketReplication configuration rules', () => { }, }); - itSkipIfE2E('should accept configuration when \'StorageClass\' is ' + - `${storageClass}`, () => checkError(config, null)); + itSkipIfE2E("should accept configuration when 'StorageClass' is " + `${storageClass}`, () => + checkError(config, null), + ); }); - it('should not accept configuration when \'StorageClass\' is invalid', - () => { - const config = setConfigRules({ - Destination: { - Bucket: `arn:aws:s3:::${destinationBucket}`, - StorageClass: 'INVALID', - }, - }); - return checkError(config, 'MalformedXML'); + it("should not accept configuration when 'StorageClass' is invalid", () => { + const config = setConfigRules({ + Destination: { + Bucket: `arn:aws:s3:::${destinationBucket}`, + StorageClass: 'INVALID', + }, }); + return checkError(config, 'MalformedXML'); + }); }); describe('aws-node-sdk test putBucketReplication CORS', () => { @@ -414,20 +450,26 @@ describe('aws-node-sdk test putBucketReplication CORS', () => { const config = getConfig('default', { signatureVersion: 'v4' }); s3 = new S3Client(config); await s3.send(new CreateBucketCommand({ Bucket: bucket })); - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Enabled' }, - })); - await s3.send(new PutBucketCorsCommand({ - Bucket: bucket, - CORSConfiguration: { - CORSRules: [{ - AllowedOrigins: ['*'], - AllowedMethods: ['PUT'], - AllowedHeaders: ['*'], - }], - }, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ); + await s3.send( + new PutBucketCorsCommand({ + Bucket: bucket, + CORSConfiguration: { + CORSRules: [ + { + AllowedOrigins: ['*'], + AllowedMethods: ['PUT'], + AllowedHeaders: ['*'], + }, + ], + }, + }), + ); }); afterEach(async () => { @@ -445,8 +487,7 @@ describe('aws-node-sdk test putBucketReplication CORS', () => { const replicationParams = { Bucket: bucket, ReplicationConfiguration: { - Role: 'arn:aws:iam::account-id:role/src-resource,' + - 'arn:aws:iam::account-id:role/dest-resource', + Role: 'arn:aws:iam::account-id:role/src-resource,' + 'arn:aws:iam::account-id:role/dest-resource', Rules: [], }, }; @@ -463,7 +504,7 @@ describe('aws-node-sdk test putBucketReplication CORS', () => { name: 'injectOriginHeader', step: 'build', priority: 'high', - } + }, ); try { diff --git a/tests/functional/aws-node-sdk/test/bucket/putBucketTagging.js b/tests/functional/aws-node-sdk/test/bucket/putBucketTagging.js index 35d26e1d15..d682355e65 100644 --- a/tests/functional/aws-node-sdk/test/bucket/putBucketTagging.js +++ b/tests/functional/aws-node-sdk/test/bucket/putBucketTagging.js @@ -1,9 +1,11 @@ const assert = require('assert'); -const { S3Client, +const { + S3Client, CreateBucketCommand, DeleteBucketCommand, PutBucketTaggingCommand, - GetBucketTaggingCommand } = require('@aws-sdk/client-s3'); + GetBucketTaggingCommand, +} = require('@aws-sdk/client-s3'); const assertError = require('../../../../utilities/bucketTagging-util'); const getConfig = require('../support/config'); @@ -52,7 +54,8 @@ const validEmptyTagging = { const taggingKeyNotValid = { TagSet: [ { - Key: 'stringaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaastringaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + + Key: + 'stringaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaastringaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + 'astringaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaastringaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + 'stringaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', Value: 'string', @@ -72,7 +75,8 @@ const taggingValueNotValid = { }, { Key: 'string', - Value: 'stringaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaastringaaaaaaaaaaaaaaaaaaaaaa' + + Value: + 'stringaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaastringaaaaaaaaaaaaaaaaaaaaaa' + 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaastringaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + 'aaaaaaastringaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaastringaaaaaaaaaaaaaaaaaa' + 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaastringaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + @@ -97,11 +101,13 @@ describe('aws-sdk test put bucket tagging', () => { it('should not add tag if tagKey not unique', async () => { try { - await s3.send(new PutBucketTaggingCommand({ - AccountId: s3.AccountId, - Tagging: taggingNotUnique, - Bucket: bucket, - })); + await s3.send( + new PutBucketTaggingCommand({ + AccountId: s3.AccountId, + Tagging: taggingNotUnique, + Bucket: bucket, + }), + ); throw new Error('Expected InvalidTag error'); } catch (err) { assertError(err, 'InvalidTag'); @@ -110,11 +116,13 @@ describe('aws-sdk test put bucket tagging', () => { it('should not add tag if tagKey not valid', async () => { try { - await s3.send(new PutBucketTaggingCommand({ - AccountId: s3.AccountId, - Tagging: taggingKeyNotValid, - Bucket: bucket, - })); + await s3.send( + new PutBucketTaggingCommand({ + AccountId: s3.AccountId, + Tagging: taggingKeyNotValid, + Bucket: bucket, + }), + ); throw new Error('Expected InvalidTag error'); } catch (err) { assertError(err, 'InvalidTag'); @@ -123,11 +131,13 @@ describe('aws-sdk test put bucket tagging', () => { it('should not add tag if tagValue not valid', async () => { try { - await s3.send(new PutBucketTaggingCommand({ - AccountId: s3.AccountId, - Tagging: taggingValueNotValid, - Bucket: bucket, - })); + await s3.send( + new PutBucketTaggingCommand({ + AccountId: s3.AccountId, + Tagging: taggingValueNotValid, + Bucket: bucket, + }), + ); throw new Error('Expected InvalidTag error'); } catch (err) { assertError(err, 'InvalidTag'); @@ -136,42 +146,54 @@ describe('aws-sdk test put bucket tagging', () => { it('should add tag', async () => { // Put bucket tagging - await s3.send(new PutBucketTaggingCommand({ - AccountId: s3.AccountId, - Tagging: validTagging, - Bucket: bucket, - })); - const res = await s3.send(new GetBucketTaggingCommand({ - AccountId: s3.AccountId, - Bucket: bucket, - })); + await s3.send( + new PutBucketTaggingCommand({ + AccountId: s3.AccountId, + Tagging: validTagging, + Bucket: bucket, + }), + ); + const res = await s3.send( + new GetBucketTaggingCommand({ + AccountId: s3.AccountId, + Bucket: bucket, + }), + ); assert.deepStrictEqual(res.TagSet, validTagging.TagSet); }); it('should be able to put single tag', async () => { - await s3.send(new PutBucketTaggingCommand({ - AccountId: s3.AccountId, - Tagging: validSingleTagging, - Bucket: bucket, - })); - const res = await s3.send(new GetBucketTaggingCommand({ - AccountId: s3.AccountId, - Bucket: bucket, - })); + await s3.send( + new PutBucketTaggingCommand({ + AccountId: s3.AccountId, + Tagging: validSingleTagging, + Bucket: bucket, + }), + ); + const res = await s3.send( + new GetBucketTaggingCommand({ + AccountId: s3.AccountId, + Bucket: bucket, + }), + ); assert.deepStrictEqual(res.TagSet, validSingleTagging.TagSet); }); it('should be able to put empty tag array', async () => { - await s3.send(new PutBucketTaggingCommand({ - AccountId: s3.AccountId, - Tagging: validEmptyTagging, - Bucket: bucket, - })); - try { - await s3.send(new GetBucketTaggingCommand({ + await s3.send( + new PutBucketTaggingCommand({ AccountId: s3.AccountId, + Tagging: validEmptyTagging, Bucket: bucket, - })); + }), + ); + try { + await s3.send( + new GetBucketTaggingCommand({ + AccountId: s3.AccountId, + Bucket: bucket, + }), + ); throw new Error('Expected NoSuchTagSet error'); } catch (err) { assertError(err, 'NoSuchTagSet'); @@ -180,12 +202,14 @@ describe('aws-sdk test put bucket tagging', () => { it('should return accessDenied if expected bucket owner does not match', async () => { try { - await s3.send(new PutBucketTaggingCommand({ - AccountId: s3.AccountId, - Tagging: validEmptyTagging, - Bucket: bucket, - ExpectedBucketOwner: '944690102203' - })); + await s3.send( + new PutBucketTaggingCommand({ + AccountId: s3.AccountId, + Tagging: validEmptyTagging, + Bucket: bucket, + ExpectedBucketOwner: '944690102203', + }), + ); throw new Error('Expected AccessDenied error'); } catch (err) { assertError(err, 'AccessDenied'); @@ -193,17 +217,21 @@ describe('aws-sdk test put bucket tagging', () => { }); it('should not return accessDenied if expected bucket owner matches', async () => { - await s3.send(new PutBucketTaggingCommand({ - AccountId: s3.AccountId, - Tagging: validEmptyTagging, - Bucket: bucket, - ExpectedBucketOwner: s3.AccountId - })); + await s3.send( + new PutBucketTaggingCommand({ + AccountId: s3.AccountId, + Tagging: validEmptyTagging, + Bucket: bucket, + ExpectedBucketOwner: s3.AccountId, + }), + ); try { - await s3.send(new GetBucketTaggingCommand({ - AccountId: s3.AccountId, - Bucket: bucket - })); + await s3.send( + new GetBucketTaggingCommand({ + AccountId: s3.AccountId, + Bucket: bucket, + }), + ); throw new Error('Expected NoSuchTagSet error'); } catch (err) { assertError(err, 'NoSuchTagSet'); @@ -217,12 +245,14 @@ describe('aws-sdk test put bucket tagging', () => { Value: `value_${index}`, })), }; - await s3.send(new PutBucketTaggingCommand({ - AccountId: s3.AccountId, - Tagging: tags, - Bucket: bucket, - ExpectedBucketOwner: s3.AccountId - })); + await s3.send( + new PutBucketTaggingCommand({ + AccountId: s3.AccountId, + Tagging: tags, + Bucket: bucket, + ExpectedBucketOwner: s3.AccountId, + }), + ); }); it('should not put more than 50 tags', async () => { @@ -233,12 +263,14 @@ describe('aws-sdk test put bucket tagging', () => { })), }; try { - await s3.send(new PutBucketTaggingCommand({ - AccountId: s3.AccountId, - Tagging: tags, - Bucket: bucket, - ExpectedBucketOwner: s3.AccountId - })); + await s3.send( + new PutBucketTaggingCommand({ + AccountId: s3.AccountId, + Tagging: tags, + Bucket: bucket, + ExpectedBucketOwner: s3.AccountId, + }), + ); throw new Error('Expected BadRequest error'); } catch (err) { assertError(err, 'BadRequest'); diff --git a/tests/functional/aws-node-sdk/test/bucket/putCors.js b/tests/functional/aws-node-sdk/test/bucket/putCors.js index 99cf6fe346..26247eea10 100644 --- a/tests/functional/aws-node-sdk/test/bucket/putCors.js +++ b/tests/functional/aws-node-sdk/test/bucket/putCors.js @@ -1,25 +1,23 @@ const assert = require('assert'); -const { S3Client, - CreateBucketCommand, - DeleteBucketCommand, - PutBucketCorsCommand } = require('@aws-sdk/client-s3'); +const { S3Client, CreateBucketCommand, DeleteBucketCommand, PutBucketCorsCommand } = require('@aws-sdk/client-s3'); const withV4 = require('../support/withV4'); const getConfig = require('../support/config'); const bucketName = 'testcorsbucket'; -const sampleCors = { CORSRules: [ - { AllowedMethods: ['PUT', 'POST', 'DELETE'], - AllowedOrigins: ['http://www.example.com'], - AllowedHeaders: ['*'], - MaxAgeSeconds: 3000, - ExposeHeaders: ['x-amz-server-side-encryption'] }, - { AllowedMethods: ['GET'], - AllowedOrigins: ['*'], - AllowedHeaders: ['*'], - MaxAgeSeconds: 3000 }, -] }; +const sampleCors = { + CORSRules: [ + { + AllowedMethods: ['PUT', 'POST', 'DELETE'], + AllowedOrigins: ['http://www.example.com'], + AllowedHeaders: ['*'], + MaxAgeSeconds: 3000, + ExposeHeaders: ['x-amz-server-side-encryption'], + }, + { AllowedMethods: ['GET'], AllowedOrigins: ['*'], AllowedHeaders: ['*'], MaxAgeSeconds: 3000 }, + ], +}; function _corsTemplate(params) { const sampleRule = { @@ -29,12 +27,11 @@ function _corsTemplate(params) { MaxAgeSeconds: 3000, ExposeHeaders: ['x-amz-server-side-encryption'], }; - ['AllowedMethods', 'AllowedOrigins', 'AllowedHeaders', 'MaxAgeSeconds', - 'ExposeHeaders'].forEach(prop => { - if (params[prop]) { - sampleRule[prop] = params[prop]; - } - }); + ['AllowedMethods', 'AllowedOrigins', 'AllowedHeaders', 'MaxAgeSeconds', 'ExposeHeaders'].forEach(prop => { + if (params[prop]) { + sampleRule[prop] = params[prop]; + } + }); return { CORSRules: [sampleRule] }; } @@ -45,10 +42,12 @@ describe('PUT bucket cors', () => { async function _testPutBucketCors(rules, statusCode, errMsg) { try { - await s3.send(new PutBucketCorsCommand({ - Bucket: bucketName, - CORSConfiguration: rules - })); + await s3.send( + new PutBucketCorsCommand({ + Bucket: bucketName, + CORSConfiguration: rules, + }), + ); throw new Error('Expected error but found none'); } catch (err) { assert.strictEqual(err.name, errMsg); @@ -56,15 +55,17 @@ describe('PUT bucket cors', () => { } } - beforeEach(() => s3.send(new CreateBucketCommand({ Bucket: bucketName }))); + beforeEach(() => s3.send(new CreateBucketCommand({ Bucket: bucketName }))); afterEach(() => s3.send(new DeleteBucketCommand({ Bucket: bucketName }))); it('should put a bucket cors successfully', async () => { - await s3.send(new PutBucketCorsCommand({ - Bucket: bucketName, - CORSConfiguration: sampleCors - })); + await s3.send( + new PutBucketCorsCommand({ + Bucket: bucketName, + CORSConfiguration: sampleCors, + }), + ); }); it('should return InvalidRequest if more than 100 rules', async () => { @@ -87,10 +88,8 @@ describe('PUT bucket cors', () => { await _testPutBucketCors(testCors, 400, 'MalformedXML'); }); - it('should return InvalidRequest if more than one asterisk in ' + - 'AllowedOrigin', async () => { - const testCors = - _corsTemplate({ AllowedOrigins: ['http://*.*.com'] }); + it('should return InvalidRequest if more than one asterisk in ' + 'AllowedOrigin', async () => { + const testCors = _corsTemplate({ AllowedOrigins: ['http://*.*.com'] }); await _testPutBucketCors(testCors, 400, 'InvalidRequest'); }); @@ -99,29 +98,28 @@ describe('PUT bucket cors', () => { await _testPutBucketCors(testCors, 400, 'MalformedXML'); }); - it('should return InvalidRequest if AllowedMethod is not a valid ' + - 'method', async () => { + it('should return InvalidRequest if AllowedMethod is not a valid ' + 'method', async () => { const testCors = _corsTemplate({ AllowedMethods: ['test'] }); await _testPutBucketCors(testCors, 400, 'InvalidRequest'); }); - it('should return InvalidRequest for lowercase value for ' + - 'AllowedMethod', async () => { + it('should return InvalidRequest for lowercase value for ' + 'AllowedMethod', async () => { const testCors = _corsTemplate({ AllowedMethods: ['put', 'get'] }); await _testPutBucketCors(testCors, 400, 'InvalidRequest'); }); - it('should return InvalidRequest if more than one asterisk in ' + - 'AllowedHeader', async () => { + it('should return InvalidRequest if more than one asterisk in ' + 'AllowedHeader', async () => { const testCors = _corsTemplate({ AllowedHeaders: ['*-amz-*'] }); await _testPutBucketCors(testCors, 400, 'InvalidRequest'); }); - it('should return InvalidRequest if ExposeHeader has character ' + - 'that is not dash or alphanumeric', async () => { - const testCors = _corsTemplate({ ExposeHeaders: ['test header'] }); - await _testPutBucketCors(testCors, 400, 'InvalidRequest'); - }); + it( + 'should return InvalidRequest if ExposeHeader has character ' + 'that is not dash or alphanumeric', + async () => { + const testCors = _corsTemplate({ ExposeHeaders: ['test header'] }); + await _testPutBucketCors(testCors, 400, 'InvalidRequest'); + }, + ); it('should return InvalidRequest if ExposeHeader has wildcard', async () => { const testCors = _corsTemplate({ ExposeHeaders: ['x-amz-*'] }); diff --git a/tests/functional/aws-node-sdk/test/bucket/putWebsite.js b/tests/functional/aws-node-sdk/test/bucket/putWebsite.js index 0d2e8181a3..adc53c2de4 100644 --- a/tests/functional/aws-node-sdk/test/bucket/putWebsite.js +++ b/tests/functional/aws-node-sdk/test/bucket/putWebsite.js @@ -1,8 +1,5 @@ const assert = require('assert'); -const { - CreateBucketCommand, - PutBucketWebsiteCommand, -} = require('@aws-sdk/client-s3'); +const { CreateBucketCommand, PutBucketWebsiteCommand } = require('@aws-sdk/client-s3'); const withV4 = require('../support/withV4'); const BucketUtility = require('../../lib/utility/bucket-util'); @@ -16,18 +13,17 @@ describe('PUT bucket website', () => { const s3 = bucketUtil.s3; function _testPutBucketWebsite(config, statusCode, errMsg, cb) { - s3.send(new PutBucketWebsiteCommand({ Bucket: bucketName, - WebsiteConfiguration: config })) - .then(() => { - cb(new Error('Expected err but found none')); - }) - .catch(err => { - assert.strictEqual(err.name, errMsg); - assert.strictEqual(err.$metadata.httpStatusCode, statusCode); - cb(); - }); + s3.send(new PutBucketWebsiteCommand({ Bucket: bucketName, WebsiteConfiguration: config })) + .then(() => { + cb(new Error('Expected err but found none')); + }) + .catch(err => { + assert.strictEqual(err.name, errMsg); + assert.strictEqual(err.$metadata.httpStatusCode, statusCode); + cb(); + }); } - beforeEach(() => s3.send(new CreateBucketCommand({ Bucket: bucketName }))); + beforeEach(() => s3.send(new CreateBucketCommand({ Bucket: bucketName }))); afterEach(async () => { await bucketUtil.empty(bucketName); @@ -36,58 +32,59 @@ describe('PUT bucket website', () => { it('should put a bucket website successfully', () => { const config = new WebsiteConfigTester('index.html'); - s3.send(new PutBucketWebsiteCommand({ Bucket: bucketName, - WebsiteConfiguration: config })); + s3.send(new PutBucketWebsiteCommand({ Bucket: bucketName, WebsiteConfiguration: config })); }); - it('should return InvalidArgument if IndexDocument or ' + - 'RedirectAllRequestsTo is not provided', done => { + it('should return InvalidArgument if IndexDocument or ' + 'RedirectAllRequestsTo is not provided', done => { const config = new WebsiteConfigTester(); _testPutBucketWebsite(config, 400, 'InvalidArgument', done); }); - it('should return an InvalidRequest if both ' + - 'RedirectAllRequestsTo and IndexDocument are provided', done => { - const redirectAllTo = { - HostName: 'test', - Protocol: 'http', - }; - const config = new WebsiteConfigTester(null, null, - redirectAllTo); - config.addRoutingRule({ Protocol: 'http' }); - _testPutBucketWebsite(config, 400, 'InvalidRequest', done); - }); + it( + 'should return an InvalidRequest if both ' + 'RedirectAllRequestsTo and IndexDocument are provided', + done => { + const redirectAllTo = { + HostName: 'test', + Protocol: 'http', + }; + const config = new WebsiteConfigTester(null, null, redirectAllTo); + config.addRoutingRule({ Protocol: 'http' }); + _testPutBucketWebsite(config, 400, 'InvalidRequest', done); + }, + ); it('should return InvalidArgument if index has slash', done => { const config = new WebsiteConfigTester('in/dex.html'); _testPutBucketWebsite(config, 400, 'InvalidArgument', done); }); - it('should return InvalidRequest if both ReplaceKeyWith and ' + - 'ReplaceKeyPrefixWith are present in same rule', done => { - const config = new WebsiteConfigTester('index.html'); - config.addRoutingRule({ ReplaceKeyPrefixWith: 'test', - ReplaceKeyWith: 'test' }); - _testPutBucketWebsite(config, 400, 'InvalidRequest', done); - }); - - it('should return InvalidRequest if both ReplaceKeyWith and ' + - 'ReplaceKeyPrefixWith are present in same rule', done => { - const config = new WebsiteConfigTester('index.html'); - config.addRoutingRule({ ReplaceKeyPrefixWith: 'test', - ReplaceKeyWith: 'test' }); - _testPutBucketWebsite(config, 400, 'InvalidRequest', done); - }); - - it('should return InvalidRequest if Redirect Protocol is ' + - 'not http or https', done => { + it( + 'should return InvalidRequest if both ReplaceKeyWith and ' + + 'ReplaceKeyPrefixWith are present in same rule', + done => { + const config = new WebsiteConfigTester('index.html'); + config.addRoutingRule({ ReplaceKeyPrefixWith: 'test', ReplaceKeyWith: 'test' }); + _testPutBucketWebsite(config, 400, 'InvalidRequest', done); + }, + ); + + it( + 'should return InvalidRequest if both ReplaceKeyWith and ' + + 'ReplaceKeyPrefixWith are present in same rule', + done => { + const config = new WebsiteConfigTester('index.html'); + config.addRoutingRule({ ReplaceKeyPrefixWith: 'test', ReplaceKeyWith: 'test' }); + _testPutBucketWebsite(config, 400, 'InvalidRequest', done); + }, + ); + + it('should return InvalidRequest if Redirect Protocol is ' + 'not http or https', done => { const config = new WebsiteConfigTester('index.html'); config.addRoutingRule({ Protocol: 'notvalidprotocol' }); _testPutBucketWebsite(config, 400, 'InvalidRequest', done); }); - it('should return InvalidRequest if RedirectAllRequestsTo Protocol ' + - 'is not http or https', done => { + it('should return InvalidRequest if RedirectAllRequestsTo Protocol ' + 'is not http or https', done => { const redirectAllTo = { HostName: 'test', Protocol: 'notvalidprotocol', @@ -96,36 +93,47 @@ describe('PUT bucket website', () => { _testPutBucketWebsite(config, 400, 'InvalidRequest', done); }); - it('should return MalformedXML if Redirect HttpRedirectCode ' + - 'is a string that does not contains a number', done => { - const config = new WebsiteConfigTester('index.html'); - config.addRoutingRule({ HttpRedirectCode: 'notvalidhttpcode' }); - _testPutBucketWebsite(config, 400, 'MalformedXML', done); - }); - - it('should return InvalidRequest if Redirect HttpRedirectCode ' + - 'is not a valid http redirect code (3XX excepting 300)', done => { - const config = new WebsiteConfigTester('index.html'); - config.addRoutingRule({ HttpRedirectCode: '400' }); - _testPutBucketWebsite(config, 400, 'InvalidRequest', done); - }); - - it('should return InvalidRequest if Condition ' + - 'HttpErrorCodeReturnedEquals is a string that does ' + - ' not contain a number', done => { - const condition = { HttpErrorCodeReturnedEquals: 'notvalidcode' }; - const config = new WebsiteConfigTester('index.html'); - config.addRoutingRule({ HostName: 'test' }, condition); - _testPutBucketWebsite(config, 400, 'MalformedXML', done); - }); - - it('should return InvalidRequest if Condition ' + - 'HttpErrorCodeReturnedEquals is not a valid http' + - 'error code (4XX or 5XX)', done => { - const condition = { HttpErrorCodeReturnedEquals: '300' }; - const config = new WebsiteConfigTester('index.html'); - config.addRoutingRule({ HostName: 'test' }, condition); - _testPutBucketWebsite(config, 400, 'InvalidRequest', done); - }); + it( + 'should return MalformedXML if Redirect HttpRedirectCode ' + 'is a string that does not contains a number', + done => { + const config = new WebsiteConfigTester('index.html'); + config.addRoutingRule({ HttpRedirectCode: 'notvalidhttpcode' }); + _testPutBucketWebsite(config, 400, 'MalformedXML', done); + }, + ); + + it( + 'should return InvalidRequest if Redirect HttpRedirectCode ' + + 'is not a valid http redirect code (3XX excepting 300)', + done => { + const config = new WebsiteConfigTester('index.html'); + config.addRoutingRule({ HttpRedirectCode: '400' }); + _testPutBucketWebsite(config, 400, 'InvalidRequest', done); + }, + ); + + it( + 'should return InvalidRequest if Condition ' + + 'HttpErrorCodeReturnedEquals is a string that does ' + + ' not contain a number', + done => { + const condition = { HttpErrorCodeReturnedEquals: 'notvalidcode' }; + const config = new WebsiteConfigTester('index.html'); + config.addRoutingRule({ HostName: 'test' }, condition); + _testPutBucketWebsite(config, 400, 'MalformedXML', done); + }, + ); + + it( + 'should return InvalidRequest if Condition ' + + 'HttpErrorCodeReturnedEquals is not a valid http' + + 'error code (4XX or 5XX)', + done => { + const condition = { HttpErrorCodeReturnedEquals: '300' }; + const config = new WebsiteConfigTester('index.html'); + config.addRoutingRule({ HostName: 'test' }, condition); + _testPutBucketWebsite(config, 400, 'InvalidRequest', done); + }, + ); }); }); diff --git a/tests/functional/aws-node-sdk/test/bucket/skipScan.js b/tests/functional/aws-node-sdk/test/bucket/skipScan.js index e846b8d713..fb4f13555d 100644 --- a/tests/functional/aws-node-sdk/test/bucket/skipScan.js +++ b/tests/functional/aws-node-sdk/test/bucket/skipScan.js @@ -1,9 +1,11 @@ -const { S3Client, +const { + S3Client, CreateBucketCommand, DeleteBucketCommand, PutObjectCommand, ListObjectsCommand, - DeleteObjectCommand } = require('@aws-sdk/client-s3'); + DeleteObjectCommand, +} = require('@aws-sdk/client-s3'); const assert = require('assert'); const getConfig = require('../support/config'); @@ -45,7 +47,7 @@ describe('Skip scan cases tests', () => { let s3; before(async () => { const config = getConfig('default', { signatureVersion: 'v4' }); - s3 = new S3Client(config); + s3 = new S3Client(config); await s3.send(new CreateBucketCommand({ Bucket })); const x = 120; const promises = []; @@ -54,30 +56,27 @@ describe('Skip scan cases tests', () => { const o = {}; o.Bucket = Bucket; // eslint-disable-next-line - o.Key = String.fromCharCode(65 + n / x) + - '/' + n % x; + o.Key = String.fromCharCode(65 + n / x) + '/' + (n % x); o.Body = ''; await s3.send(new PutObjectCommand(o)); }; promises.push(putObjectPromise); - } + } for (let i = 0; i < promises.length; i += 10) { const batch = promises.slice(i, i + 10); await Promise.all(batch.map(fn => fn())); } }); - + after(async () => { const data = await s3.send(new ListObjectsCommand({ Bucket })); - const deletePromises = data.Contents.map(o => - s3.send(new DeleteObjectCommand({ Bucket, Key: o.Key })) - ); + const deletePromises = data.Contents.map(o => s3.send(new DeleteObjectCommand({ Bucket, Key: o.Key }))); await Promise.all(deletePromises); await s3.send(new DeleteBucketCommand({ Bucket })); }); - + it('should find all common prefixes in one shot', async () => { - const { $metadata , ...data } = await s3.send(new ListObjectsCommand({ Bucket, Delimiter: '/' })); + const { $metadata, ...data } = await s3.send(new ListObjectsCommand({ Bucket, Delimiter: '/' })); cutAttributes(data); assert.deepStrictEqual(data, { IsTruncated: false, @@ -86,13 +85,7 @@ describe('Skip scan cases tests', () => { Name: Bucket, Prefix: '', MaxKeys: 1000, - CommonPrefixes: [ - 'A/', - 'B/', - 'C/', - 'D/', - 'E/', - ], + CommonPrefixes: ['A/', 'B/', 'C/', 'D/', 'E/'], }); assert.strictEqual($metadata.httpStatusCode, 200); }); diff --git a/tests/functional/aws-node-sdk/test/bucket/testBucketStress.js b/tests/functional/aws-node-sdk/test/bucket/testBucketStress.js index a9b37b792a..bb4e3dd643 100644 --- a/tests/functional/aws-node-sdk/test/bucket/testBucketStress.js +++ b/tests/functional/aws-node-sdk/test/bucket/testBucketStress.js @@ -1,8 +1,10 @@ -const { S3Client, +const { + S3Client, CreateBucketCommand, DeleteBucketCommand, PutObjectCommand, - DeleteObjectCommand } = require('@aws-sdk/client-s3'); + DeleteObjectCommand, +} = require('@aws-sdk/client-s3'); const getConfig = require('../support/config'); diff --git a/tests/functional/aws-node-sdk/test/bucket/testBucketVersioning.js b/tests/functional/aws-node-sdk/test/bucket/testBucketVersioning.js index 09ea95d26d..56ed756abe 100644 --- a/tests/functional/aws-node-sdk/test/bucket/testBucketVersioning.js +++ b/tests/functional/aws-node-sdk/test/bucket/testBucketVersioning.js @@ -1,16 +1,17 @@ const assert = require('assert'); -const { S3Client, +const { + S3Client, CreateBucketCommand, DeleteBucketCommand, PutBucketVersioningCommand, - GetBucketVersioningCommand } = require('@aws-sdk/client-s3'); + GetBucketVersioningCommand, +} = require('@aws-sdk/client-s3'); const getConfig = require('../support/config'); const bucket = `versioning-bucket-${Date.now()}`; const config = getConfig('default', { signatureVersion: 'v4' }); -const configReplication = getConfig('replication', - { signatureVersion: 'v4' }); +const configReplication = getConfig('replication', { signatureVersion: 'v4' }); const s3 = new S3Client(config); describe('aws-node-sdk test bucket versioning', function testSuite() { this.timeout(60000); @@ -33,8 +34,7 @@ describe('aws-node-sdk test bucket versioning', function testSuite() { throw new Error('accepted empty versioning configuration'); } catch (error) { assert.strictEqual(error.$metadata.httpStatusCode, 400); - assert.strictEqual( - error.name, 'IllegalVersioningConfigurationException'); + assert.strictEqual(error.name, 'IllegalVersioningConfigurationException'); } }); @@ -57,8 +57,7 @@ describe('aws-node-sdk test bucket versioning', function testSuite() { throw new Error('accepted empty versioning configuration'); } catch (error) { assert.strictEqual(error.$metadata.httpStatusCode, 400); - assert.strictEqual( - error.name, 'IllegalVersioningConfigurationException'); + assert.strictEqual(error.name, 'IllegalVersioningConfigurationException'); } }); @@ -74,7 +73,7 @@ describe('aws-node-sdk test bucket versioning', function testSuite() { Bucket: bucket, VersioningConfiguration: { MFADelete: 'fun', - Status: 'let\'s do it', + Status: "let's do it", }, }; try { @@ -82,8 +81,7 @@ describe('aws-node-sdk test bucket versioning', function testSuite() { throw new Error('accepted empty versioning configuration'); } catch (error) { assert.strictEqual(error.$metadata.httpStatusCode, 400); - assert.strictEqual( - error.name, 'IllegalVersioningConfigurationException'); + assert.strictEqual(error.name, 'IllegalVersioningConfigurationException'); } }); @@ -142,8 +140,7 @@ describe('aws-node-sdk test bucket versioning', function testSuite() { // S3C doesn't support service account. There is no cross account access for replication account. // (canonicalId looking like http://acs.zenko.io/accounts/service/replication) const itSkipS3C = process.env.S3_END_TO_END ? it.skip : it; - itSkipS3C('should accept valid versioning configuration if user is a ' + - 'replication user', async () => { + itSkipS3C('should accept valid versioning configuration if user is a ' + 'replication user', async () => { const params = { Bucket: bucket, VersioningConfiguration: { @@ -160,26 +157,31 @@ describe('aws-node-sdk test bucket versioning', function testSuite() { }); }); - describe('bucket versioning for ingestion buckets', () => { const Bucket = `ingestion-bucket-${Date.now()}`; - before(() => s3.send(new CreateBucketCommand({ - Bucket, - CreateBucketConfiguration: { - LocationConstraint: 'us-east-2:ingest', - }, - }))); + before(() => + s3.send( + new CreateBucketCommand({ + Bucket, + CreateBucketConfiguration: { + LocationConstraint: 'us-east-2:ingest', + }, + }), + ), + ); after(() => s3.send(new DeleteBucketCommand({ Bucket }))); it('should not allow suspending versioning for ingestion buckets', async () => { try { - await s3.send(new PutBucketVersioningCommand({ - Bucket, - VersioningConfiguration: { - Status: 'Suspended' - } - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket, + VersioningConfiguration: { + Status: 'Suspended', + }, + }), + ); throw new Error('Expected error but got success'); } catch (err) { assert.strictEqual(err.name, 'InvalidBucketState'); @@ -193,10 +195,12 @@ describe('aws-node-sdk test bucket versioning with object lock', () => { before(async () => { const config = getConfig('default', { signatureVersion: 'v4' }); s3ObjectLock = new S3Client(config); - await s3ObjectLock.send(new CreateBucketCommand({ - Bucket: bucket, - ObjectLockEnabledForBucket: true, - })); + await s3ObjectLock.send( + new CreateBucketCommand({ + Bucket: bucket, + ObjectLockEnabledForBucket: true, + }), + ); }); after(() => s3ObjectLock.send(new DeleteBucketCommand({ Bucket: bucket }))); @@ -216,4 +220,3 @@ describe('aws-node-sdk test bucket versioning with object lock', () => { } }); }); - diff --git a/tests/functional/aws-node-sdk/test/bucket/updateBucketQuota.js b/tests/functional/aws-node-sdk/test/bucket/updateBucketQuota.js index fc14716652..68e622269d 100644 --- a/tests/functional/aws-node-sdk/test/bucket/updateBucketQuota.js +++ b/tests/functional/aws-node-sdk/test/bucket/updateBucketQuota.js @@ -1,6 +1,4 @@ -const { S3Client, - CreateBucketCommand, - DeleteBucketCommand } = require('@aws-sdk/client-s3'); +const { S3Client, CreateBucketCommand, DeleteBucketCommand } = require('@aws-sdk/client-s3'); const assert = require('assert'); const getConfig = require('../support/config'); @@ -25,19 +23,21 @@ describe('Test update bucket quota', () => { afterEach(() => s3.send(new DeleteBucketCommand({ Bucket: bucket }))); - it('should update the quota, using json parsing by default', () => sendRequest('PUT', - '127.0.0.1:8000', `/${bucket}/?quota=true`, JSON.stringify(quota))); + it('should update the quota, using json parsing by default', () => + sendRequest('PUT', '127.0.0.1:8000', `/${bucket}/?quota=true`, JSON.stringify(quota))); it('should update quota with explicit JSON content-type', async () => { - await sendRequest('PUT', '127.0.0.1:8000', `/${bucket}/?quota=true`, - JSON.stringify(quota), null, new Date(), { 'Content-Type': 'application/json' }); + await sendRequest('PUT', '127.0.0.1:8000', `/${bucket}/?quota=true`, JSON.stringify(quota), null, new Date(), { + 'Content-Type': 'application/json', + }); }); it('should update quota with XML format', async () => { try { const xmlQuota = '3000'; - await sendRequest('PUT', '127.0.0.1:8000', `/${bucket}/?quota=true`, - xmlQuota, null, new Date(), { 'Content-Type': 'application/xml' }); + await sendRequest('PUT', '127.0.0.1:8000', `/${bucket}/?quota=true`, xmlQuota, null, new Date(), { + 'Content-Type': 'application/xml', + }); assert.ok(true); } catch (err) { assert.fail(`Expected no error, but got ${err}`); @@ -70,6 +70,6 @@ describe('Test update bucket quota', () => { } }); - it('should accept large quota', () => sendRequest('PUT', - '127.0.0.1:8000', `/${bucket}/?quota=true`, JSON.stringify(largeQuota))); + it('should accept large quota', () => + sendRequest('PUT', '127.0.0.1:8000', `/${bucket}/?quota=true`, JSON.stringify(largeQuota))); }); diff --git a/tests/functional/aws-node-sdk/test/legacy/authV2QueryTests.js b/tests/functional/aws-node-sdk/test/legacy/authV2QueryTests.js index 683857f1d7..0536b160a8 100644 --- a/tests/functional/aws-node-sdk/test/legacy/authV2QueryTests.js +++ b/tests/functional/aws-node-sdk/test/legacy/authV2QueryTests.js @@ -19,10 +19,7 @@ const provideRawOutputAsync = util.promisify(provideRawOutput); const random = Math.round(Math.random() * 100).toString(); const bucket = `mybucket-${random}`; const almostOutsideTime = 99990; -const itSkipAWS = process.env.AWS_ON_AIR - ? it.skip - : it; - +const itSkipAWS = process.env.AWS_ON_AIR ? it.skip : it; function diff(putFile, receivedFile, done) { process.stdout.write(`diff ${putFile} ${receivedFile}\n`); @@ -51,13 +48,11 @@ describe('aws-node-sdk v2auth query tests', function testSuite() { // AWS allows an expiry further in the future // 604810 seconds is higher that the Expires time limit: 604800 seconds // ( seven days) - itSkipAWS('should return an error code if expires header is too far ' + - 'in the future', async () => { - + itSkipAWS('should return an error code if expires header is too far ' + 'in the future', async () => { // First, get a valid signed URL with maximum allowed expiry const command = new CreateBucketCommand({ Bucket: bucket }); const validUrl = await getSignedUrl(s3, command, { expiresIn: 604800 }); // Exactly 7 days - + // Manually modify the URL to have a longer expiry const urlObj = new URL(validUrl); const futureExpiry = Math.floor(Date.now() / 1000) + 604810; // 10 seconds more than limit @@ -82,12 +77,10 @@ describe('aws-node-sdk v2auth query tests', function testSuite() { assert.strictEqual(httpCode, '200 OK'); }); - it('should put an object', async () => { const command = new PutObjectCommand({ Bucket: bucket, Key: 'key' }); const url = await getSignedUrl(s3, command, { expiresIn: almostOutsideTime }); - const { httpCode } = await provideRawOutputAsync(['-verbose', '-X', 'PUT', url, - '--upload-file', 'uploadFile']); + const { httpCode } = await provideRawOutputAsync(['-verbose', '-X', 'PUT', url, '--upload-file', 'uploadFile']); assert.strictEqual(httpCode, '200 OK'); }); @@ -97,19 +90,17 @@ describe('aws-node-sdk v2auth query tests', function testSuite() { // This will also test that query params that contain "x-amz-" // are being added to the canonical headers list in our string // to sign. - const command = new PutObjectCommand({ - Bucket: bucket, + const command = new PutObjectCommand({ + Bucket: bucket, Key: 'key', - ACL: 'public-read', - StorageClass: 'STANDARD' + ACL: 'public-read', + StorageClass: 'STANDARD', }); const url = await getSignedUrl(s3, command); - const { httpCode } = await provideRawOutputAsync(['-verbose', '-X', 'PUT', url, - '--upload-file', 'uploadFile']); + const { httpCode } = await provideRawOutputAsync(['-verbose', '-X', 'PUT', url, '--upload-file', 'uploadFile']); assert.strictEqual(httpCode, '200 OK'); }); - it('should get an object', async () => { const command = new GetObjectCommand({ Bucket: bucket, Key: 'key' }); const url = await getSignedUrl(s3, command, { expiresIn: almostOutsideTime }); @@ -130,7 +121,6 @@ describe('aws-node-sdk v2auth query tests', function testSuite() { assert.strictEqual(httpCode, '204 NO CONTENT'); }); - it('should delete a bucket', async () => { const command = new DeleteBucketCommand({ Bucket: bucket }); const url = await getSignedUrl(s3, command, { expiresIn: almostOutsideTime }); diff --git a/tests/functional/aws-node-sdk/test/legacy/authV4QueryTests.js b/tests/functional/aws-node-sdk/test/legacy/authV4QueryTests.js index 23df1352cd..3142d8148b 100644 --- a/tests/functional/aws-node-sdk/test/legacy/authV4QueryTests.js +++ b/tests/functional/aws-node-sdk/test/legacy/authV4QueryTests.js @@ -55,11 +55,7 @@ describe('aws-node-sdk v4auth query tests', function testSuite() { it('should create a bucket', async () => { const params = { Bucket: bucket }; - const url = await getSignedUrl( - s3, - new CreateBucketCommand(params), - { expiresIn: 900 } - ); + const url = await getSignedUrl(s3, new CreateBucketCommand(params), { expiresIn: 900 }); const { httpCode } = await provideRawOutputAsync(['-verbose', '-X', 'PUT', url]); assert.strictEqual(httpCode, '200 OK'); }); @@ -69,16 +65,14 @@ describe('aws-node-sdk v4auth query tests', function testSuite() { const { httpCode, rawOutput } = await provideRawOutputAsync(['-verbose', url]); assert.strictEqual(httpCode, '200 OK'); const xml = await parseStringAsync(rawOutput.stdout); - const bucketNames = xml.ListAllMyBucketsResult - .Buckets[0].Bucket.map(item => item.Name[0]); + const bucketNames = xml.ListAllMyBucketsResult.Buckets[0].Bucket.map(item => item.Name[0]); assert(bucketNames.indexOf(bucket) > -1); }); it('should put an object', async () => { const params = { Bucket: bucket, Key: 'key' }; const url = await getSignedUrl(s3, new PutObjectCommand(params), { expiresIn: 900 }); - const { httpCode } = await provideRawOutputAsync(['-verbose', '-X', 'PUT', url, - '--upload-file', 'uploadFile']); + const { httpCode } = await provideRawOutputAsync(['-verbose', '-X', 'PUT', url, '--upload-file', 'uploadFile']); assert.strictEqual(httpCode, '200 OK'); }); @@ -91,18 +85,17 @@ describe('aws-node-sdk v4auth query tests', function testSuite() { ContentType: 'text/plain', }; const url = await getSignedUrl(s3, new PutObjectCommand(params), { expiresIn: 900 }); - const { httpCode } = await provideRawOutputAsync(['-verbose', '-X', 'PUT', url, - '--upload-file', 'uploadFile']); + const { httpCode } = await provideRawOutputAsync(['-verbose', '-X', 'PUT', url, '--upload-file', 'uploadFile']); assert.strictEqual(httpCode, '200 OK'); }); it('should put an object with native characters', async () => { - const Key = 'key-pâtisserie-中文-español-English-हिन्दी-العربية-' + - 'português-বাংলা-русский-日本語-ਪੰਜਾਬੀ-한국어-தமிழ்'; + const Key = + 'key-pâtisserie-中文-español-English-हिन्दी-العربية-' + + 'português-বাংলা-русский-日本語-ਪੰਜਾਬੀ-한국어-தமிழ்'; const params = { Bucket: bucket, Key }; const url = await getSignedUrl(s3, new PutObjectCommand(params), { expiresIn: 900 }); - const { httpCode } = await provideRawOutputAsync(['-verbose', '-X', 'PUT', url, - '--upload-file', 'uploadFile']); + const { httpCode } = await provideRawOutputAsync(['-verbose', '-X', 'PUT', url, '--upload-file', 'uploadFile']); assert.strictEqual(httpCode, '200 OK'); }); @@ -153,8 +146,9 @@ describe('aws-node-sdk v4auth query tests', function testSuite() { }); it('should delete an object with native characters', async () => { - const Key = 'key-pâtisserie-中文-español-English-हिन्दी-العربية-' + - 'português-বাংলা-русский-日本語-ਪੰਜਾਬੀ-한국어-தமிழ்'; + const Key = + 'key-pâtisserie-中文-español-English-हिन्दी-العربية-' + + 'português-বাংলা-русский-日本語-ਪੰਜਾਬੀ-한국어-தமிழ்'; const params = { Bucket: bucket, Key }; const url = await getSignedUrl(s3, new DeleteObjectCommand(params), { expiresIn: 900 }); const { httpCode } = await provideRawOutputAsync(['-verbose', '-X', 'DELETE', url]); diff --git a/tests/functional/aws-node-sdk/test/legacy/tests.js b/tests/functional/aws-node-sdk/test/legacy/tests.js index fd704f9014..ed9e4535c3 100644 --- a/tests/functional/aws-node-sdk/test/legacy/tests.js +++ b/tests/functional/aws-node-sdk/test/legacy/tests.js @@ -156,21 +156,24 @@ describe('aws-node-sdk test suite as registered user', function testSuite() { assert.strictEqual(data.StorageClass, 'STANDARD'); }); - it('should return an error if do not provide correct ' + - // completempu test - 'xml when completing a multipart upload', async () => { - const params = { - Bucket: bucket, - Key: 'toComplete', - UploadId: multipartUploadData.secondUploadId, - }; - try { - await s3.send(new CompleteMultipartUploadCommand(params)); - throw new Error('Expected MalformedXML error'); - } catch (err) { - assert.strictEqual(err.Code, 'MalformedXML'); - } - }); + it( + 'should return an error if do not provide correct ' + + // completempu test + 'xml when completing a multipart upload', + async () => { + const params = { + Bucket: bucket, + Key: 'toComplete', + UploadId: multipartUploadData.secondUploadId, + }; + try { + await s3.send(new CompleteMultipartUploadCommand(params)); + throw new Error('Expected MalformedXML error'); + } catch (err) { + assert.strictEqual(err.Code, 'MalformedXML'); + } + }, + ); // completempu test it('should complete a multipart upload', async () => { @@ -214,8 +217,8 @@ describe('aws-node-sdk test suite as registered user', function testSuite() { }); const mpuRangeGetTests = [ - { it: 'should get a range from the first part of an object ' + - 'put by multipart upload', + { + it: 'should get a range from the first part of an object ' + 'put by multipart upload', range: 'bytes=0-9', contentLength: 10, contentRange: 'bytes 0-9/10485760', @@ -224,8 +227,8 @@ describe('aws-node-sdk test suite as registered user', function testSuite() { // first part should just contain 0 expectedBuff: Buffer.alloc(10, 0), }, - { it: 'should get a range from the second part of an object ' + - 'put by multipart upload', + { + it: 'should get a range from the second part of an object ' + 'put by multipart upload', // The completed MPU byte count starts at 0, so the first part ends // at byte 5242879 and the second part begins at byte 5242880 range: 'bytes=5242880-5242889', @@ -234,8 +237,8 @@ describe('aws-node-sdk test suite as registered user', function testSuite() { // A range from the second part should just contain 1 expectedBuff: Buffer.alloc(10, 1), }, - { it: 'should get a range that spans both parts of an object put ' + - 'by multipart upload', + { + it: 'should get a range that spans both parts of an object put ' + 'by multipart upload', range: 'bytes=5242875-5242884', contentLength: 10, contentRange: 'bytes 5242875-5242884/10485760', @@ -243,9 +246,11 @@ describe('aws-node-sdk test suite as registered user', function testSuite() { // of 0 and 5 bytes of 1 expectedBuff: Buffer.allocUnsafe(10).fill(0, 0, 5).fill(1, 5, 10), }, - { it: 'should get a range from the second part of an object put by ' + - 'multipart upload and include the end even if the range ' + - 'requested goes beyond the actual object end', + { + it: + 'should get a range from the second part of an object put by ' + + 'multipart upload and include the end even if the range ' + + 'requested goes beyond the actual object end', // End is actually 10485759 since size is 10485760 range: 'bytes=10485750-10485790', contentLength: 10, @@ -303,8 +308,7 @@ describe('aws-node-sdk test suite as registered user', function testSuite() { assert.ok(data); }); - it('should return InvalidRange if the range of the resource does ' + - 'not cover the byte range', async () => { + it('should return InvalidRange if the range of the resource does ' + 'not cover the byte range', async () => { const params = { Bucket: bucket, Key: 'normalput', @@ -333,8 +337,7 @@ describe('aws-node-sdk test suite as registered user', function testSuite() { }); testsRangeOnEmptyFile.forEach(test => { const validText = test.valid ? 'InvalidRange error' : 'empty file'; - it(`should return ${validText} if get range ${test.range} on ` + - 'empty object', async () => { + it(`should return ${validText} if get range ${test.range} on ` + 'empty object', async () => { const getParams = { Bucket: bucketEmptyObj, Key: 'emptyobj', @@ -363,28 +366,30 @@ describe('aws-node-sdk test suite as registered user', function testSuite() { }); const regularObjectRangeGetTests = [ - { it: 'should get a range for an object put without MPU', + { + it: 'should get a range for an object put without MPU', range: 'bytes=10-99', contentLength: 90, contentRange: 'bytes 10-99/200', // Buffer.fill(value, offset, end) expectedBuff: Buffer.allocUnsafe(90).fill(0, 0, 40).fill(1, 40), }, - { it: 'should get a range for an object using only an end ' + - 'offset in the request', + { + it: 'should get a range for an object using only an end ' + 'offset in the request', range: 'bytes=-10', contentLength: 10, contentRange: 'bytes 190-199/200', expectedBuff: Buffer.alloc(10, 1), }, - { it: 'should get a range for an object using only a start offset ' + - 'in the request', + { + it: 'should get a range for an object using only a start offset ' + 'in the request', range: 'bytes=190-', contentLength: 10, contentRange: 'bytes 190-199/200', expectedBuff: Buffer.alloc(10, 1), }, - { it: 'should get full object if range header is invalid', + { + it: 'should get full object if range header is invalid', range: 'bytes=-', contentLength: 200, // Since range header is invalid full object should be returned diff --git a/tests/functional/aws-node-sdk/test/mdSearch/basicSearch.js b/tests/functional/aws-node-sdk/test/mdSearch/basicSearch.js index d97876bc92..4166e01649 100644 --- a/tests/functional/aws-node-sdk/test/mdSearch/basicSearch.js +++ b/tests/functional/aws-node-sdk/test/mdSearch/basicSearch.js @@ -16,136 +16,130 @@ const updatedUserMetadata = { food: 'cake' }; runIfMongo('Basic search', () => { const bucketName = `basicsearchmebucket${Date.now()}`; - + before(async () => { await s3Client.send(new CreateBucketCommand({ Bucket: bucketName })); - await s3Client.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectKey, - Metadata: userMetadata, - Tagging: objectTagData, - })); - await s3Client.send(new PutObjectCommand({ - Bucket: bucketName, - Key: hiddenKey, - Tagging: hiddenTagData, - })); + await s3Client.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectKey, + Metadata: userMetadata, + Tagging: objectTagData, + }), + ); + await s3Client.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: hiddenKey, + Tagging: hiddenTagData, + }), + ); }); after(async () => { - await s3Client.send(new DeleteObjectsCommand({ - Bucket: bucketName, - Delete: { - Objects: [ - { Key: objectKey }, - { Key: hiddenKey }, - ], - }, - })); + await s3Client.send( + new DeleteObjectsCommand({ + Bucket: bucketName, + Delete: { + Objects: [{ Key: objectKey }, { Key: hiddenKey }], + }, + }), + ); await s3Client.send(new DeleteBucketCommand({ Bucket: bucketName })); }); it('should list object with searched for system metadata', done => { const encodedSearch = encodeURIComponent(`key="${objectKey}"`); - return runAndCheckSearch(s3Client, bucketName, - encodedSearch, false, objectKey, done); + return runAndCheckSearch(s3Client, bucketName, encodedSearch, false, objectKey, done); }); it('should list object with regex searched for system metadata', done => { const encodedSearch = encodeURIComponent('key LIKE "find.*"'); - return runAndCheckSearch(s3Client, bucketName, - encodedSearch, false, objectKey, done); + return runAndCheckSearch(s3Client, bucketName, encodedSearch, false, objectKey, done); }); - it('should list object with regex searched for system metadata with flags', - done => { + it('should list object with regex searched for system metadata with flags', done => { const encodedSearch = encodeURIComponent('key LIKE "/FIND.*/i"'); - return runAndCheckSearch(s3Client, bucketName, - encodedSearch, false, objectKey, done); + return runAndCheckSearch(s3Client, bucketName, encodedSearch, false, objectKey, done); }); it('should return empty when no object match regex', done => { const encodedSearch = encodeURIComponent('key LIKE "/NOTFOUND.*/i"'); - return runAndCheckSearch(s3Client, bucketName, - encodedSearch, false, null, done); + return runAndCheckSearch(s3Client, bucketName, encodedSearch, false, null, done); }); it('should list object with searched for user metadata', done => { - const encodedSearch = - encodeURIComponent(`x-amz-meta-food="${userMetadata.food}"`); - return runAndCheckSearch(s3Client, bucketName, - encodedSearch, false, objectKey, done); + const encodedSearch = encodeURIComponent(`x-amz-meta-food="${userMetadata.food}"`); + return runAndCheckSearch(s3Client, bucketName, encodedSearch, false, objectKey, done); }); it('should list object with searched for tag metadata', done => { - const encodedSearch = - encodeURIComponent('tags.item-type="main"'); - return runAndCheckSearch(s3Client, bucketName, - encodedSearch, false, objectKey, done); + const encodedSearch = encodeURIComponent('tags.item-type="main"'); + return runAndCheckSearch(s3Client, bucketName, encodedSearch, false, objectKey, done); }); it('should return empty listing when no object has user md', done => { - const encodedSearch = - encodeURIComponent('x-amz-meta-food="nosuchfood"'); - return runAndCheckSearch(s3Client, bucketName, - encodedSearch, false, null, done); + const encodedSearch = encodeURIComponent('x-amz-meta-food="nosuchfood"'); + return runAndCheckSearch(s3Client, bucketName, encodedSearch, false, null, done); }); describe('search when overwrite object', () => { before(done => { - s3Client.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectKey, - Metadata: updatedUserMetadata, - })) + s3Client + .send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectKey, + Metadata: updatedUserMetadata, + }), + ) .then(() => done()) .catch(done); }); - it('should list object with searched for updated user metadata', - done => { - const encodedSearch = - encodeURIComponent('x-amz-meta-food' + - `="${updatedUserMetadata.food}"`); - return runAndCheckSearch(s3Client, bucketName, - encodedSearch, false, objectKey, done); - }); + it('should list object with searched for updated user metadata', done => { + const encodedSearch = encodeURIComponent('x-amz-meta-food' + `="${updatedUserMetadata.food}"`); + return runAndCheckSearch(s3Client, bucketName, encodedSearch, false, objectKey, done); + }); }); }); runIfMongo('Search when no objects in bucket', () => { const bucketName = `noobjectbucket${Date.now()}`; - + before(done => { - s3Client.send(new CreateBucketCommand({ Bucket: bucketName })) + s3Client + .send(new CreateBucketCommand({ Bucket: bucketName })) .then(() => done()) .catch(done); }); after(done => { - s3Client.send(new DeleteBucketCommand({ Bucket: bucketName })) + s3Client + .send(new DeleteBucketCommand({ Bucket: bucketName })) .then(() => done()) .catch(done); }); it('should return empty listing when no objects in bucket', done => { const encodedSearch = encodeURIComponent(`key="${objectKey}"`); - return runAndCheckSearch(s3Client, bucketName, - encodedSearch, false, null, done); + return runAndCheckSearch(s3Client, bucketName, encodedSearch, false, null, done); }); }); runIfMongo('Invalid regular expression searches', () => { const bucketName = `badregex-${Date.now()}`; - + before(done => { - s3Client.send(new CreateBucketCommand({ Bucket: bucketName })) + s3Client + .send(new CreateBucketCommand({ Bucket: bucketName })) .then(() => done()) .catch(done); }); after(done => { - s3Client.send(new DeleteBucketCommand({ Bucket: bucketName })) + s3Client + .send(new DeleteBucketCommand({ Bucket: bucketName })) .then(() => done()) .catch(done); }); @@ -156,7 +150,6 @@ runIfMongo('Invalid regular expression searches', () => { code: 'InvalidArgument', message: 'Invalid sql where clause sent as search query', }; - return runAndCheckSearch(s3Client, bucketName, - encodedSearch, false, testError, done); + return runAndCheckSearch(s3Client, bucketName, encodedSearch, false, testError, done); }); }); diff --git a/tests/functional/aws-node-sdk/test/mdSearch/utils/helpers.js b/tests/functional/aws-node-sdk/test/mdSearch/utils/helpers.js index a3b14b2c77..1e882d8431 100644 --- a/tests/functional/aws-node-sdk/test/mdSearch/utils/helpers.js +++ b/tests/functional/aws-node-sdk/test/mdSearch/utils/helpers.js @@ -1,10 +1,6 @@ const assert = require('assert'); const async = require('async'); -const { - ListObjectsCommand, - ListObjectVersionsCommand, - DeleteObjectsCommand, -} = require('@aws-sdk/client-s3'); +const { ListObjectsCommand, ListObjectVersionsCommand, DeleteObjectsCommand } = require('@aws-sdk/client-s3'); function _deleteVersionList(s3Client, versionList, bucket, callback) { if (versionList === undefined || versionList.length === 0) { @@ -19,31 +15,30 @@ function _deleteVersionList(s3Client, versionList, bucket, callback) { })), }, }; - return s3Client.send(new DeleteObjectsCommand(params)) + return s3Client + .send(new DeleteObjectsCommand(params)) .then(() => callback()) .catch(callback); } const testUtils = {}; -testUtils.runIfMongo = process.env.S3METADATA === 'mongodb' ? - describe : describe.skip; +testUtils.runIfMongo = process.env.S3METADATA === 'mongodb' ? describe : describe.skip; -testUtils.runAndCheckSearch = (s3Client, bucketName, encodedSearch, listVersions, - testResult, done) => { +testUtils.runAndCheckSearch = (s3Client, bucketName, encodedSearch, listVersions, testResult, done) => { const makeRequest = async () => { try { - const input = { + const input = { Bucket: bucketName, }; - + let command; if (listVersions) { command = new ListObjectVersionsCommand(input); } else { command = new ListObjectsCommand(input); } - + // Add middleware to inject the search query parameter // SDK v3 automatically encodes query parameters, so we decode first to avoid double-encoding command.middlewareStack.add( @@ -59,26 +54,30 @@ testUtils.runAndCheckSearch = (s3Client, bucketName, encodedSearch, listVersions // eslint-disable-next-line no-param-reassign args.request.query.versions = ''; } - + return next(args); }, { step: 'build', name: 'addSearchQuery', - } + }, ); - + const res = await s3Client.send(command); - + if (listVersions) { if (testResult) { assert.notStrictEqual(res.Versions[0].VersionId, undefined); if (Array.isArray(testResult)) { assert.strictEqual(res.Versions.length, testResult.length); - async.forEachOf(testResult, (expected, i, next) => { - assert.strictEqual(res.Versions[i].Key, expected); - next(); - }, done); + async.forEachOf( + testResult, + (expected, i, next) => { + assert.strictEqual(res.Versions[i].Key, expected); + next(); + }, + done, + ); } else { assert(res.Versions[0], 'should be Contents listed'); assert.strictEqual(res.Versions[0].Key, testResult); @@ -114,26 +113,29 @@ testUtils.runAndCheckSearch = (s3Client, bucketName, encodedSearch, listVersions }; testUtils.removeAllVersions = (s3Client, bucket, callback) => { - async.waterfall([ - cb => s3Client.send(new ListObjectVersionsCommand({ Bucket: bucket })) - .then(data => cb(null, data)) - .catch(cb), - (data, cb) => _deleteVersionList(s3Client, data.DeleteMarkers, bucket, - err => cb(err, data)), - (data, cb) => _deleteVersionList(s3Client, data.Versions, bucket, - err => cb(err, data)), - (data, cb) => { - if (data.IsTruncated) { - const params = { - Bucket: bucket, - KeyMarker: data.NextKeyMarker, - VersionIdMarker: data.NextVersionIdMarker, - }; - return testUtils.removeAllVersions(s3Client, params, cb); - } - return cb(); - }, - ], callback); + async.waterfall( + [ + cb => + s3Client + .send(new ListObjectVersionsCommand({ Bucket: bucket })) + .then(data => cb(null, data)) + .catch(cb), + (data, cb) => _deleteVersionList(s3Client, data.DeleteMarkers, bucket, err => cb(err, data)), + (data, cb) => _deleteVersionList(s3Client, data.Versions, bucket, err => cb(err, data)), + (data, cb) => { + if (data.IsTruncated) { + const params = { + Bucket: bucket, + KeyMarker: data.NextKeyMarker, + VersionIdMarker: data.NextVersionIdMarker, + }; + return testUtils.removeAllVersions(s3Client, params, cb); + } + return cb(); + }, + ], + callback, + ); }; module.exports = testUtils; diff --git a/tests/functional/aws-node-sdk/test/mdSearch/versionEnabledSearch.js b/tests/functional/aws-node-sdk/test/mdSearch/versionEnabledSearch.js index 904f3b5696..7e83b6f75d 100644 --- a/tests/functional/aws-node-sdk/test/mdSearch/versionEnabledSearch.js +++ b/tests/functional/aws-node-sdk/test/mdSearch/versionEnabledSearch.js @@ -6,8 +6,7 @@ const { } = require('@aws-sdk/client-s3'); const { promisify } = require('util'); const s3Client = require('./utils/s3SDK'); -const { runAndCheckSearch, removeAllVersions, runIfMongo } = - require('./utils/helpers'); +const { runAndCheckSearch, removeAllVersions, runIfMongo } = require('./utils/helpers'); const removeAllVersionsPromise = promisify(removeAllVersions); const userMetadata = { food: 'pizza' }; const updatedMetadata = { food: 'pineapple' }; @@ -21,48 +20,50 @@ runIfMongo('Search in version enabled bucket', () => { }; before(async () => { await s3Client.send(new CreateBucketCommand({ Bucket: bucketName })); - await s3Client.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration - })); - await s3Client.send(new PutObjectCommand({ - Bucket: bucketName, - Key: masterKey, - Metadata: userMetadata - })); + await s3Client.send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration, + }), + ); + await s3Client.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: masterKey, + Metadata: userMetadata, + }), + ); }); after(async () => { - await removeAllVersionsPromise(s3Client, bucketName); - await s3Client.send(new DeleteBucketCommand({ Bucket: bucketName })); + await removeAllVersionsPromise(s3Client, bucketName); + await s3Client.send(new DeleteBucketCommand({ Bucket: bucketName })); }); it('should list just master object with searched for metadata by default', done => { - const encodedSearch = - encodeURIComponent(`x-amz-meta-food="${userMetadata.food}"`); + const encodedSearch = encodeURIComponent(`x-amz-meta-food="${userMetadata.food}"`); runAndCheckSearch(s3Client, bucketName, encodedSearch, false, masterKey, done); }); describe('New version overwrite', () => { before(async () => { - await s3Client.send(new PutObjectCommand({ - Bucket: bucketName, - Key: masterKey, - Metadata: updatedMetadata - })); + await s3Client.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: masterKey, + Metadata: updatedMetadata, + }), + ); }); it('should list just master object with updated metadata by default', done => { - const encodedSearch = - encodeURIComponent(`x-amz-meta-food="${updatedMetadata.food}"`); + const encodedSearch = encodeURIComponent(`x-amz-meta-food="${updatedMetadata.food}"`); runAndCheckSearch(s3Client, bucketName, encodedSearch, false, masterKey, done); }); it('should list all object versions that met search query while specifying versions param', done => { - const encodedSearch = - encodeURIComponent('x-amz-meta-food LIKE "pi.*"'); - runAndCheckSearch(s3Client, bucketName, - encodedSearch, true, [masterKey, masterKey], done); + const encodedSearch = encodeURIComponent('x-amz-meta-food LIKE "pi.*"'); + runAndCheckSearch(s3Client, bucketName, encodedSearch, true, [masterKey, masterKey], done); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/acl/aclAwsVersioning.js b/tests/functional/aws-node-sdk/test/multipleBackend/acl/aclAwsVersioning.js index a47d6f784c..bd26091b45 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/acl/aclAwsVersioning.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/acl/aclAwsVersioning.js @@ -86,7 +86,7 @@ function putObjectAndAcl(s3, key, body, acp, cb) { Key: key, Body: body, }); - + s3.send(command) .then(putData => { putObjectAcl(s3, key, putData.VersionId, acp, err => { @@ -113,20 +113,24 @@ function putVersionsWithAclToAws(s3, key, data, acps, cb) { throw new Error('length of data and acp arrays must be the same'); } enableVersioning(s3, bucket, () => { - async.timesLimit(data.length, 1, (i, next) => { - putObjectAndAcl(s3, key, data[i], acps[i], next); - }, (err, results) => { - if (err) { - return cb(err); - } - return cb(null, results); - }); + async.timesLimit( + data.length, + 1, + (i, next) => { + putObjectAndAcl(s3, key, data[i], acps[i], next); + }, + (err, results) => { + if (err) { + return cb(err); + } + return cb(null, results); + }, + ); }); } function getObjectAndAssertAcl(s3, params, cb) { - const { bucket, key, versionId, body, expectedVersionId, expectedResult } - = params; + const { bucket, key, versionId, body, expectedVersionId, expectedResult } = params; getAndAssertResult(s3, { bucket, key, versionId, expectedVersionId, body }) .then(() => { const aclParams = { @@ -136,13 +140,13 @@ function getObjectAndAssertAcl(s3, params, cb) { if (versionId) { aclParams.VersionId = versionId; } - + const command = new GetObjectAclCommand(aclParams); return s3.send(command); }) .then(data => { // eslint-disable-next-line no-unused-vars - const {$metadata, ...aclData} = data; + const { $metadata, ...aclData } = data; assert.deepEqual(aclData, expectedResult); cb(); }) @@ -160,23 +164,28 @@ function getObjectAndAssertAcl(s3, params, cb) { * @param {function} cb - callback * @return {undefined} - and call cb */ -function getObjectsAndAssertAcls(s3, key, versionIds, expectedData, - expectedAcps, cb) { - async.timesLimit(versionIds.length, 1, (i, next) => { - const versionId = versionIds[i]; - const body = expectedData[i]; - const expectedResult = expectedAcps[i]; - getObjectAndAssertAcl(s3, { bucket, key, versionId, body, - expectedResult, expectedVersionId: versionId }, next); - }, err => { - assert.strictEqual(err, null, 'Expected success ' + - `getting object acls, got error ${err}`); - cb(); - }); +function getObjectsAndAssertAcls(s3, key, versionIds, expectedData, expectedAcps, cb) { + async.timesLimit( + versionIds.length, + 1, + (i, next) => { + const versionId = versionIds[i]; + const body = expectedData[i]; + const expectedResult = expectedAcps[i]; + getObjectAndAssertAcl( + s3, + { bucket, key, versionId, body, expectedResult, expectedVersionId: versionId }, + next, + ); + }, + err => { + assert.strictEqual(err, null, 'Expected success ' + `getting object acls, got error ${err}`); + cb(); + }, + ); } -describeSkipIfNotMultiple('AWS backend put/get object acl with versioning', -function testSuite() { +describeSkipIfNotMultiple('AWS backend put/get object acl with versioning', function testSuite() { this.timeout(30000); withV4(sigCfg => { let bucketUtil; @@ -186,37 +195,35 @@ function testSuite() { process.stdout.write('Creating bucket'); bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; - + const command = new CreateBucketCommand({ Bucket: bucket, CreateBucketConfiguration: { LocationConstraint: awsLocation, }, }); - - return s3.send(command) - .catch(err => { - process.stdout.write(`Error creating bucket: ${err}\n`); - throw err; - }); + + return s3.send(command).catch(err => { + process.stdout.write(`Error creating bucket: ${err}\n`); + throw err; + }); }); afterEach(() => { process.stdout.write('Emptying bucket\n'); - return bucketUtil.empty(bucket) - .then(() => { - process.stdout.write('Deleting bucket\n'); - return bucketUtil.deleteOne(bucket); - }) - .catch(err => { - process.stdout.write('Error emptying/deleting bucket: ' + - `${err}\n`); - throw err; - }); + return bucketUtil + .empty(bucket) + .then(() => { + process.stdout.write('Deleting bucket\n'); + return bucketUtil.deleteOne(bucket); + }) + .catch(err => { + process.stdout.write('Error emptying/deleting bucket: ' + `${err}\n`); + throw err; + }); }); - it('versioning not configured: should put/get acl successfully when ' + - 'versioning not configured', done => { + it('versioning not configured: should put/get acl successfully when ' + 'versioning not configured', done => { const key = `somekey-${genUniqID()}`; waitForVersioningBeforePut(s3, bucket, err => { if (err) { @@ -224,85 +231,97 @@ function testSuite() { } return putObjectAndAcl(s3, key, someBody, testAcp, (err, versionId) => { assert.strictEqual(versionId, undefined); - getObjectAndAssertAcl(s3, { bucket, key, body: someBody, - expectedResult: testAcp }, done); + getObjectAndAssertAcl(s3, { bucket, key, body: someBody, expectedResult: testAcp }, done); }); }); }); - it('versioning suspended then enabled: should put/get acl on null ' + - 'version successfully even when latest version is not null version', - done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => putNullVersionsToAws(s3, bucket, key, [undefined], - err => next(err)), - next => putVersionsToAws(s3, bucket, key, [someBody], - err => next(err)), - next => { - putObjectAcl(s3, key, 'null', testAcp, next); - }, - next => getObjectAndAssertAcl(s3, { bucket, key, body: '', - versionId: 'null', expectedResult: testAcp, - expectedVersionId: 'null' }, next), - ], done); - }); + it( + 'versioning suspended then enabled: should put/get acl on null ' + + 'version successfully even when latest version is not null version', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => putNullVersionsToAws(s3, bucket, key, [undefined], err => next(err)), + next => putVersionsToAws(s3, bucket, key, [someBody], err => next(err)), + next => { + putObjectAcl(s3, key, 'null', testAcp, next); + }, + next => + getObjectAndAssertAcl( + s3, + { + bucket, + key, + body: '', + versionId: 'null', + expectedResult: testAcp, + expectedVersionId: 'null', + }, + next, + ), + ], + done, + ); + }, + ); - it('versioning enabled: should get correct acl using version IDs', - done => { + it('versioning enabled: should get correct acl using version IDs', done => { const key = `somekey-${genUniqID()}`; - const acps = ['READ', 'FULL_CONTROL', 'READ_ACP', 'WRITE_ACP'] - .map(perm => { + const acps = ['READ', 'FULL_CONTROL', 'READ_ACP', 'WRITE_ACP'].map(perm => { const acp = new _AccessControlPolicy(ownerParams); acp.addGrantee('Group', constants.publicId, perm); return acp; }); const data = [...Array(acps.length).keys()].map(i => i.toString()); const versionIds = ['null']; - async.waterfall([ - next => { - putObjectAndAcl(s3, key, data[0], acps[0], - () => next()); - }, - next => { - putVersionsWithAclToAws(s3, key, data.slice(1), - acps.slice(1), next); - }, - (ids, next) => { - versionIds.push(...ids); - next(); - }, - next => { - getObjectsAndAssertAcls(s3, key, versionIds, data, acps, - next); - }, - ], done); + async.waterfall( + [ + next => { + putObjectAndAcl(s3, key, data[0], acps[0], () => next()); + }, + next => { + putVersionsWithAclToAws(s3, key, data.slice(1), acps.slice(1), next); + }, + (ids, next) => { + versionIds.push(...ids); + next(); + }, + next => { + getObjectsAndAssertAcls(s3, key, versionIds, data, acps, next); + }, + ], + done, + ); }); - it('versioning enabled: should get correct acl when getting ' + - 'without version ID', done => { + it('versioning enabled: should get correct acl when getting ' + 'without version ID', done => { const key = `somekey-${genUniqID()}`; - const acps = ['READ', 'FULL_CONTROL', 'READ_ACP', 'WRITE_ACP'] - .map(perm => { + const acps = ['READ', 'FULL_CONTROL', 'READ_ACP', 'WRITE_ACP'].map(perm => { const acp = new _AccessControlPolicy(ownerParams); acp.addGrantee('Group', constants.publicId, perm); return acp; }); const data = [...Array(acps.length).keys()].map(i => i.toString()); const versionIds = ['null']; - async.waterfall([ - next => putObjectAndAcl(s3, key, data[0], acps[0], - () => next()), - next => putVersionsWithAclToAws(s3, key, data.slice(1), - acps.slice(1), next), - (ids, next) => { - versionIds.push(...ids); - next(); - }, - next => getObjectAndAssertAcl(s3, { bucket, key, - expectedVersionId: versionIds[3], - expectedResult: acps[3], body: data[3] }, next), - ], done); + async.waterfall( + [ + next => putObjectAndAcl(s3, key, data[0], acps[0], () => next()), + next => putVersionsWithAclToAws(s3, key, data.slice(1), acps.slice(1), next), + (ids, next) => { + versionIds.push(...ids); + next(); + }, + next => + getObjectAndAssertAcl( + s3, + { bucket, key, expectedVersionId: versionIds[3], expectedResult: acps[3], body: data[3] }, + next, + ), + ], + done, + ); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/delete/delete.js b/tests/functional/aws-node-sdk/test/multipleBackend/delete/delete.js index 138695e931..a4e107141f 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/delete/delete.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/delete/delete.js @@ -1,10 +1,5 @@ const assert = require('assert'); -const { - CreateBucketCommand, - PutObjectCommand, - DeleteObjectCommand, - GetObjectCommand, -} = require('@aws-sdk/client-s3'); +const { CreateBucketCommand, PutObjectCommand, DeleteObjectCommand, GetObjectCommand } = require('@aws-sdk/client-s3'); const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); @@ -36,55 +31,67 @@ describeSkipIfNotMultiple('Multiple backend delete', () => { process.stdout.write('Creating bucket\n'); bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; - + await s3.send(new CreateBucketCommand({ Bucket: bucket })); - + process.stdout.write('Putting object to mem\n'); - await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: memObject, - Body: body, - Metadata: { 'scal-location-constraint': memLocation } - })); - + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: memObject, + Body: body, + Metadata: { 'scal-location-constraint': memLocation }, + }), + ); + process.stdout.write('Putting object to file\n'); - await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: fileObject, - Body: body, - Metadata: { 'scal-location-constraint': fileLocation } - })); - + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: fileObject, + Body: body, + Metadata: { 'scal-location-constraint': fileLocation }, + }), + ); + process.stdout.write('Putting object to AWS\n'); - await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: awsObject, - Body: body, - Metadata: { 'scal-location-constraint': awsLocation } - })); - + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: awsObject, + Body: body, + Metadata: { 'scal-location-constraint': awsLocation }, + }), + ); + process.stdout.write('Putting 0-byte object to AWS\n'); - await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: emptyObject, - Metadata: { 'scal-location-constraint': awsLocation } - })); - + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: emptyObject, + Metadata: { 'scal-location-constraint': awsLocation }, + }), + ); + process.stdout.write('Putting large object to AWS\n'); - await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: bigObject, - Body: bigBody, - Metadata: { 'scal-location-constraint': awsLocation } - })); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: bigObject, + Body: bigBody, + Metadata: { 'scal-location-constraint': awsLocation }, + }), + ); process.stdout.write('Putting object to AWS\n'); - await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: mismatchObject, - Body: body, - Metadata: { 'scal-location-constraint': awsLocationMismatch } - })); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: mismatchObject, + Body: body, + Metadata: { 'scal-location-constraint': awsLocationMismatch }, + }), + ); }); after(async () => { @@ -106,7 +113,7 @@ describeSkipIfNotMultiple('Multiple backend delete', () => { it('should delete object from file', async () => { await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: fileObject })); - + try { await s3.send(new GetObjectCommand({ Bucket: bucket, Key: fileObject })); assert.fail('Expected NoSuchKey error but got success'); @@ -117,7 +124,7 @@ describeSkipIfNotMultiple('Multiple backend delete', () => { it('should delete an object from AWS', async () => { await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: awsObject })); - + try { await s3.send(new GetObjectCommand({ Bucket: bucket, Key: awsObject })); assert.fail('Expected NoSuchKey error but got success'); @@ -128,7 +135,7 @@ describeSkipIfNotMultiple('Multiple backend delete', () => { it('should delete 0-byte object from AWS', async () => { await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: emptyObject })); - + try { await s3.send(new GetObjectCommand({ Bucket: bucket, Key: emptyObject })); assert.fail('Expected NoSuchKey error but got success'); @@ -139,7 +146,7 @@ describeSkipIfNotMultiple('Multiple backend delete', () => { it('should delete large object from AWS', async () => { await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: bigObject })); - + try { await s3.send(new GetObjectCommand({ Bucket: bucket, Key: bigObject })); assert.fail('Expected NoSuchKey error but got success'); @@ -148,14 +155,15 @@ describeSkipIfNotMultiple('Multiple backend delete', () => { } }); - it('should delete object from AWS location with bucketMatch set to ' + - 'false', async () => { + it('should delete object from AWS location with bucketMatch set to ' + 'false', async () => { try { await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: mismatchObject })); - await s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: mismatchObject - })); + await s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: mismatchObject, + }), + ); assert.fail('Expected NoSuchKey error but got success'); } catch (err) { assert.strictEqual(err.name, 'NoSuchKey'); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/delete/deleteAwsVersioning.js b/tests/functional/aws-node-sdk/test/multipleBackend/delete/deleteAwsVersioning.js index e8ded07b8f..fb7c7f2f08 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/delete/deleteAwsVersioning.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/delete/deleteAwsVersioning.js @@ -7,7 +7,7 @@ const { DeleteObjectsCommand, GetObjectCommand, PutObjectCommand, - GetBucketVersioningCommand + GetBucketVersioningCommand, } = require('@aws-sdk/client-s3'); const withV4 = require('../../support/withV4'); @@ -45,22 +45,23 @@ const _deleteResultSchema = { deleteDeleteMarker: [true, true, true], }; -const [nonVersionedDelete, newDeleteMarker, deleteVersion, deleteDeleteMarker] - = Object.keys(_deleteResultSchema); +const [nonVersionedDelete, newDeleteMarker, deleteVersion, deleteDeleteMarker] = Object.keys(_deleteResultSchema); function _assertDeleteResult(result, resultType, requestVersionId) { if (!_deleteResultSchema[resultType]) { throw new Error(`undefined result type "${resultType}"`); } - const [expectVersionId, matchReqVersionId, expectDeleteMarker] = - _deleteResultSchema[resultType]; + const [expectVersionId, matchReqVersionId, expectDeleteMarker] = _deleteResultSchema[resultType]; if (expectVersionId && matchReqVersionId) { assert.strictEqual(result.VersionId, requestVersionId); } else if (expectVersionId) { assert(result.VersionId, 'expected version id in result'); } else { - assert.strictEqual(result.VersionId, undefined, - `did not expect version id in result, got "${result.VersionId}"`); + assert.strictEqual( + result.VersionId, + undefined, + `did not expect version id in result, got "${result.VersionId}"`, + ); } if (expectDeleteMarker) { assert.strictEqual(result.DeleteMarker, true); @@ -71,24 +72,29 @@ function _assertDeleteResult(result, resultType, requestVersionId) { function delAndAssertResult(s3, params, cb) { const { bucket, key, versionId, resultType, resultError } = params; - return s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: versionId - })).then(result => { - if (resultError) { - assert.fail(`expected ${resultError} but got success`); - } - _assertDeleteResult(result, resultType, versionId); - return cb(null, result.VersionId); - }).catch(err => { - if (resultError) { - assert.strictEqual(err.name, resultError); - assert.strictEqual(err.$metadata.httpStatusCode, errors[resultError].code); - return cb(null); - } - return cb(err); - }); + return s3 + .send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: versionId, + }), + ) + .then(result => { + if (resultError) { + assert.fail(`expected ${resultError} but got success`); + } + _assertDeleteResult(result, resultType, versionId); + return cb(null, result.VersionId); + }) + .catch(err => { + if (resultError) { + assert.strictEqual(err.name, resultError); + assert.strictEqual(err.$metadata.httpStatusCode, errors[resultError].code); + return cb(null); + } + return cb(err); + }); } function delObjectsAndAssertResult(s3, params, cb) { @@ -102,53 +108,67 @@ function delObjectsAndAssertResult(s3, params, cb) { ], Quiet: false, }; - return s3.send(new DeleteObjectsCommand({ - Bucket: bucket, - Delete: deleteParams - })).then(res => { - if (resultError) { - assert.fail(`expected ${resultError} but got success`); - } - const result = res.Deleted[0]; - _assertDeleteResult(result, resultType, versionId); - return cb(null, result.VersionId); - }).catch(err => { - if (resultError) { - assert.strictEqual(err.name, resultError); - assert.strictEqual(err.$metadata.httpStatusCode, errors[resultError].code); - return cb(null); - } - return cb(err); - }); + return s3 + .send( + new DeleteObjectsCommand({ + Bucket: bucket, + Delete: deleteParams, + }), + ) + .then(res => { + if (resultError) { + assert.fail(`expected ${resultError} but got success`); + } + const result = res.Deleted[0]; + _assertDeleteResult(result, resultType, versionId); + return cb(null, result.VersionId); + }) + .catch(err => { + if (resultError) { + assert.strictEqual(err.name, resultError); + assert.strictEqual(err.$metadata.httpStatusCode, errors[resultError].code); + return cb(null); + } + return cb(err); + }); } function _createDeleteMarkers(s3, bucket, key, count, cb) { - return async.timesSeries(count, - (i, next) => delAndAssertResult(s3, { bucket, key, - resultType: newDeleteMarker }, next), - cb); + return async.timesSeries( + count, + (i, next) => delAndAssertResult(s3, { bucket, key, resultType: newDeleteMarker }, next), + cb, + ); } function _deleteDeleteMarkers(s3, bucket, key, deleteMarkerVids, cb) { - return async.mapSeries(deleteMarkerVids, (versionId, next) => { - delAndAssertResult(s3, { bucket, key, versionId, - resultType: deleteDeleteMarker }, next); - }, () => cb()); + return async.mapSeries( + deleteMarkerVids, + (versionId, next) => { + delAndAssertResult(s3, { bucket, key, versionId, resultType: deleteDeleteMarker }, next); + }, + () => cb(), + ); } function _getAssertDeleted(s3, params, cb) { const { key, versionId, errorCode } = params; - return s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: versionId - })).then(() => { - assert.fail('Expected error but got success'); - }).catch(err => { - assert.strictEqual(err.name, errorCode); - assert.strictEqual(err.$metadata.httpStatusCode, 404); - return cb(); - }); + return s3 + .send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: versionId, + }), + ) + .then(() => { + assert.fail('Expected error but got success'); + }) + .catch(err => { + assert.strictEqual(err.name, errorCode); + assert.strictEqual(err.$metadata.httpStatusCode, 404); + return cb(); + }); } // Update AWS S3 direct calls @@ -161,562 +181,702 @@ function _awsGetAssertDeleted(params, cb) { }); } -describeSkipIfNotMultiple('AWS backend delete object w. versioning: ' + - 'using object location constraint', function testSuite() { - this.timeout(120000); - withV4(sigCfg => { - let bucketUtil; - let s3; - beforeEach(() => { - process.stdout.write('Creating bucket\n'); - bucketUtil = new BucketUtility('default', sigCfg); - s3 = bucketUtil.s3; - return s3.send(new CreateBucketCommand({ Bucket: bucket })) - .then(() => s3.send(new GetBucketVersioningCommand({ Bucket: bucket }))) - .catch(err => { - process.stdout.write(`Error creating bucket: ${err}\n`); - throw err; +describeSkipIfNotMultiple( + 'AWS backend delete object w. versioning: ' + 'using object location constraint', + function testSuite() { + this.timeout(120000); + withV4(sigCfg => { + let bucketUtil; + let s3; + beforeEach(() => { + process.stdout.write('Creating bucket\n'); + bucketUtil = new BucketUtility('default', sigCfg); + s3 = bucketUtil.s3; + return s3 + .send(new CreateBucketCommand({ Bucket: bucket })) + .then(() => s3.send(new GetBucketVersioningCommand({ Bucket: bucket }))) + .catch(err => { + process.stdout.write(`Error creating bucket: ${err}\n`); + throw err; + }); }); - }); - afterEach(() => { - process.stdout.write('Emptying bucket\n'); - return bucketUtil.empty(bucket) - .then(() => { - process.stdout.write('Deleting bucket\n'); - return bucketUtil.deleteOne(bucket); - }) - .catch(err => { - process.stdout.write('Error emptying/deleting bucket: ' + - `${err}\n`); - throw err; + afterEach(() => { + process.stdout.write('Emptying bucket\n'); + return bucketUtil + .empty(bucket) + .then(() => { + process.stdout.write('Deleting bucket\n'); + return bucketUtil.deleteOne(bucket); + }) + .catch(err => { + process.stdout.write('Error emptying/deleting bucket: ' + `${err}\n`); + throw err; + }); }); - }); - it('versioning not configured: if specifying "null" version, should ' + - 'delete specific version in AWS backend', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => waitForVersioningBeforePut(s3, bucket, next), - next => putToAwsBackend(s3, bucket, key, someBody, - err => next(err)), - next => awsGetLatestVerId(key, someBody, next), - (awsVerId, next) => delAndAssertResult(s3, { bucket, - key, versionId: 'null', resultType: deleteVersion }, - err => next(err, awsVerId)), - (awsVerId, next) => { - _awsGetAssertDeleted({ key, - versionId: awsVerId, errorCode: 'NoSuchVersion' }, next); + it( + 'versioning not configured: if specifying "null" version, should ' + + 'delete specific version in AWS backend', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => waitForVersioningBeforePut(s3, bucket, next), + next => putToAwsBackend(s3, bucket, key, someBody, err => next(err)), + next => awsGetLatestVerId(key, someBody, next), + (awsVerId, next) => + delAndAssertResult( + s3, + { bucket, key, versionId: 'null', resultType: deleteVersion }, + err => next(err, awsVerId), + ), + (awsVerId, next) => { + _awsGetAssertDeleted({ key, versionId: awsVerId, errorCode: 'NoSuchVersion' }, next); + }, + ], + done, + ); }, - ], done); - }); - - it('versioning not configured: specifying any version id other ' + - 'than null should not result in its deletion in AWS backend', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => putToAwsBackend(s3, bucket, key, someBody, - err => next(err)), - next => awsGetLatestVerId(key, someBody, next), - (awsVerId, next) => delAndAssertResult(s3, { bucket, - key, versionId: 'awsVerIdWhichIsLongerThan40BytesButNotLongEnough', - resultError: 'InvalidArgument' }, err => next(err, awsVerId)), - (awsVerId, next) => awsGetLatestVerId(key, someBody, - (err, resultVid) => { - assert.strictEqual(resultVid, awsVerId); - next(); - }), - ], done); - }); - - it('versioning suspended: should delete a specific version in AWS ' + - 'backend successfully', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => waitForVersioningBeforePut(s3, bucket, next), - next => putNullVersionsToAws(s3, bucket, key, [someBody], - err => next(err)), - next => awsGetLatestVerId(key, someBody, next), - (awsVerId, next) => delAndAssertResult(s3, { bucket, - key, versionId: 'null', resultType: deleteVersion }, - err => next(err, awsVerId)), - (awsVerId, next) => { - _awsGetAssertDeleted({ key, - versionId: awsVerId, errorCode: 'NoSuchVersion' }, next); + ); + + it( + 'versioning not configured: specifying any version id other ' + + 'than null should not result in its deletion in AWS backend', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => putToAwsBackend(s3, bucket, key, someBody, err => next(err)), + next => awsGetLatestVerId(key, someBody, next), + (awsVerId, next) => + delAndAssertResult( + s3, + { + bucket, + key, + versionId: 'awsVerIdWhichIsLongerThan40BytesButNotLongEnough', + resultError: 'InvalidArgument', + }, + err => next(err, awsVerId), + ), + (awsVerId, next) => + awsGetLatestVerId(key, someBody, (err, resultVid) => { + assert.strictEqual(resultVid, awsVerId); + next(); + }), + ], + done, + ); }, - ], done); - }); - - it('versioning enabled: should delete a specific version in AWS ' + - 'backend successfully', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => waitForVersioningBeforePut(s3, bucket, next), - next => putVersionsToAws(s3, bucket, key, [someBody], - (err, versionIds) => next(err, versionIds[0])), - (s3vid, next) => awsGetLatestVerId(key, someBody, - (err, awsVid) => next(err, s3vid, awsVid)), - (s3VerId, awsVerId, next) => delAndAssertResult(s3, { bucket, - key, versionId: s3VerId, resultType: deleteVersion }, - err => next(err, awsVerId)), - (awsVerId, next) => { - _awsGetAssertDeleted({ key, - versionId: awsVerId, errorCode: 'NoSuchVersion' }, next); - }, - ], done); - }); - - it('versioning not configured: deleting existing object should ' + - 'not return version id or x-amz-delete-marker: true but should ' + - 'create a delete marker in aws ', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => putToAwsBackend(s3, bucket, key, someBody, - err => next(err)), - next => delAndAssertResult(s3, { bucket, key, - resultType: nonVersionedDelete }, err => next(err)), - next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' }, - next), - next => _awsGetAssertDeleted({ key, errorCode: 'NoSuchKey' }, - next), - ], done); - }); + ); + + it('versioning suspended: should delete a specific version in AWS ' + 'backend successfully', done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => waitForVersioningBeforePut(s3, bucket, next), + next => putNullVersionsToAws(s3, bucket, key, [someBody], err => next(err)), + next => awsGetLatestVerId(key, someBody, next), + (awsVerId, next) => + delAndAssertResult(s3, { bucket, key, versionId: 'null', resultType: deleteVersion }, err => + next(err, awsVerId), + ), + (awsVerId, next) => { + _awsGetAssertDeleted({ key, versionId: awsVerId, errorCode: 'NoSuchVersion' }, next); + }, + ], + done, + ); + }); - it('versioning suspended: should create a delete marker in s3 ' + - 'and aws successfully when deleting existing object', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => putNullVersionsToAws(s3, bucket, key, [someBody], - err => next(err)), - next => delAndAssertResult(s3, { bucket, key, resultType: - newDeleteMarker }, err => next(err)), - next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' }, - next), - next => _awsGetAssertDeleted({ key, errorCode: 'NoSuchKey' }, - next), - ], done); - }); + it('versioning enabled: should delete a specific version in AWS ' + 'backend successfully', done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => waitForVersioningBeforePut(s3, bucket, next), + next => + putVersionsToAws(s3, bucket, key, [someBody], (err, versionIds) => + next(err, versionIds[0]), + ), + (s3vid, next) => awsGetLatestVerId(key, someBody, (err, awsVid) => next(err, s3vid, awsVid)), + (s3VerId, awsVerId, next) => + delAndAssertResult( + s3, + { bucket, key, versionId: s3VerId, resultType: deleteVersion }, + err => next(err, awsVerId), + ), + (awsVerId, next) => { + _awsGetAssertDeleted({ key, versionId: awsVerId, errorCode: 'NoSuchVersion' }, next); + }, + ], + done, + ); + }); - // NOTE: Normal deletes when versioning is suspended create a - // delete marker with the version id "null", which overwrites an - // existing null version in s3 metadata. - it('versioning suspended: creating a delete marker will overwrite an ' + - 'existing null version that is the latest version in s3 metadata,' + - ' but the data of the first null version will remain in AWS', - function itF(done) { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => putNullVersionsToAws(s3, bucket, key, [someBody], - err => next(err)), - next => awsGetLatestVerId(key, someBody, next), - (awsNullVid, next) => { - this.test.awsNullVid = awsNullVid; - next(); + it( + 'versioning not configured: deleting existing object should ' + + 'not return version id or x-amz-delete-marker: true but should ' + + 'create a delete marker in aws ', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => putToAwsBackend(s3, bucket, key, someBody, err => next(err)), + next => + delAndAssertResult(s3, { bucket, key, resultType: nonVersionedDelete }, err => + next(err), + ), + next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' }, next), + next => _awsGetAssertDeleted({ key, errorCode: 'NoSuchKey' }, next), + ], + done, + ); }, - // following call should generate a delete marker - next => delAndAssertResult(s3, { bucket, key, resultType: - newDeleteMarker }, next), - // delete delete marker - (dmVid, next) => delAndAssertResult(s3, { bucket, key, - versionId: dmVid, resultType: deleteDeleteMarker }, - err => next(err)), - // should get no such object even after deleting del marker - next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' }, - next), - // get directly to aws however will give us first null version - next => awsGetLatestVerId(key, someBody, next), - (awsLatestVid, next) => { - assert.strictEqual(awsLatestVid, this.test.awsNullVid); - next(); + ); + + it( + 'versioning suspended: should create a delete marker in s3 ' + + 'and aws successfully when deleting existing object', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => putNullVersionsToAws(s3, bucket, key, [someBody], err => next(err)), + next => + delAndAssertResult(s3, { bucket, key, resultType: newDeleteMarker }, err => next(err)), + next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' }, next), + next => _awsGetAssertDeleted({ key, errorCode: 'NoSuchKey' }, next), + ], + done, + ); }, - ], done); - }); - - // NOTE: Normal deletes when versioning is suspended create a - // delete marker with the version id "null" which is supposed to - // overwrite any existing null version. - it('versioning suspended: creating a delete marker will overwrite an ' + - 'existing null version that is not the latest version in s3 metadata,' + - ' but the data of the first null version will remain in AWS', - function itF(done) { - const key = `somekey-${genUniqID()}`; - const data = [undefined, 'data1']; - async.waterfall([ - // put null version - next => putToAwsBackend(s3, bucket, key, data[0], - err => next(err)), - next => awsGetLatestVerId(key, '', next), - (awsNullVid, next) => { - this.test.awsNullVid = awsNullVid; - next(); + ); + + // NOTE: Normal deletes when versioning is suspended create a + // delete marker with the version id "null", which overwrites an + // existing null version in s3 metadata. + it( + 'versioning suspended: creating a delete marker will overwrite an ' + + 'existing null version that is the latest version in s3 metadata,' + + ' but the data of the first null version will remain in AWS', + function itF(done) { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => putNullVersionsToAws(s3, bucket, key, [someBody], err => next(err)), + next => awsGetLatestVerId(key, someBody, next), + (awsNullVid, next) => { + this.test.awsNullVid = awsNullVid; + next(); + }, + // following call should generate a delete marker + next => delAndAssertResult(s3, { bucket, key, resultType: newDeleteMarker }, next), + // delete delete marker + (dmVid, next) => + delAndAssertResult( + s3, + { bucket, key, versionId: dmVid, resultType: deleteDeleteMarker }, + err => next(err), + ), + // should get no such object even after deleting del marker + next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' }, next), + // get directly to aws however will give us first null version + next => awsGetLatestVerId(key, someBody, next), + (awsLatestVid, next) => { + assert.strictEqual(awsLatestVid, this.test.awsNullVid); + next(); + }, + ], + done, + ); }, - // enable versioning and put another version - next => putVersionsToAws(s3, bucket, key, [data[1]], next), - (versions, next) => { - this.test.s3vid = versions[0]; - next(); + ); + + // NOTE: Normal deletes when versioning is suspended create a + // delete marker with the version id "null" which is supposed to + // overwrite any existing null version. + it( + 'versioning suspended: creating a delete marker will overwrite an ' + + 'existing null version that is not the latest version in s3 metadata,' + + ' but the data of the first null version will remain in AWS', + function itF(done) { + const key = `somekey-${genUniqID()}`; + const data = [undefined, 'data1']; + async.waterfall( + [ + // put null version + next => putToAwsBackend(s3, bucket, key, data[0], err => next(err)), + next => awsGetLatestVerId(key, '', next), + (awsNullVid, next) => { + this.test.awsNullVid = awsNullVid; + next(); + }, + // enable versioning and put another version + next => putVersionsToAws(s3, bucket, key, [data[1]], next), + (versions, next) => { + this.test.s3vid = versions[0]; + next(); + }, + next => suspendVersioning(s3, bucket, next), + // overwrites null version in s3 metadata but does not send + // additional delete to AWS to clean up previous "null" version + next => delAndAssertResult(s3, { bucket, key, resultType: newDeleteMarker }, next), + (s3dmVid, next) => { + this.test.s3DeleteMarkerId = s3dmVid; + next(); + }, + // delete delete marker + next => + delAndAssertResult( + s3, + { + bucket, + key, + versionId: this.test.s3DeleteMarkerId, + resultType: deleteDeleteMarker, + }, + err => next(err), + ), + // deleting latest version after del marker + next => + delAndAssertResult( + s3, + { bucket, key, versionId: this.test.s3vid, resultType: deleteVersion }, + err => next(err), + ), + // should get no such object instead of null version + next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' }, next), + // we get the null version that should have been "overwritten" + // when getting the latest version in AWS now + next => awsGetLatestVerId(key, '', next), + (awsLatestVid, next) => { + assert.strictEqual(awsLatestVid, this.test.awsNullVid); + next(); + }, + ], + done, + ); }, - next => suspendVersioning(s3, bucket, next), - // overwrites null version in s3 metadata but does not send - // additional delete to AWS to clean up previous "null" version - next => delAndAssertResult(s3, { bucket, key, - resultType: newDeleteMarker }, next), - (s3dmVid, next) => { - this.test.s3DeleteMarkerId = s3dmVid; - next(); + ); + + it( + 'versioning enabled: should create a delete marker in s3 and ' + + 'aws successfully when deleting existing object', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => putVersionsToAws(s3, bucket, key, [someBody], err => next(err)), + next => + delAndAssertResult(s3, { bucket, key, resultType: newDeleteMarker }, err => next(err)), + next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' }, next), + next => _awsGetAssertDeleted({ key, errorCode: 'NoSuchKey' }, next), + ], + done, + ); }, - // delete delete marker - next => delAndAssertResult(s3, { bucket, key, - versionId: this.test.s3DeleteMarkerId, - resultType: deleteDeleteMarker }, err => next(err)), - // deleting latest version after del marker - next => delAndAssertResult(s3, { bucket, key, - versionId: this.test.s3vid, resultType: deleteVersion }, - err => next(err)), - // should get no such object instead of null version - next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' }, - next), - // we get the null version that should have been "overwritten" - // when getting the latest version in AWS now - next => awsGetLatestVerId(key, '', next), - (awsLatestVid, next) => { - assert.strictEqual(awsLatestVid, this.test.awsNullVid); - next(); - }, - ], done); - }); - - it('versioning enabled: should create a delete marker in s3 and ' + - 'aws successfully when deleting existing object', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => putVersionsToAws(s3, bucket, key, [someBody], - err => next(err)), - next => delAndAssertResult(s3, { bucket, key, resultType: - newDeleteMarker }, err => next(err)), - next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' }, - next), - next => _awsGetAssertDeleted({ key, errorCode: 'NoSuchKey' }, - next), - ], done); - }); - - it('versioning enabled: should delete a delete marker in s3 and ' + - 'aws successfully', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => waitForVersioningBeforePut(s3, bucket, next), - next => putVersionsToAws(s3, bucket, key, [someBody], - (err, versionIds) => next(err, versionIds[0])), - // create a delete marker - (s3vid, next) => delAndAssertResult(s3, { bucket, key, - resultType: newDeleteMarker }, (err, delMarkerVid) => - next(err, s3vid, delMarkerVid)), - // delete delete marker - (s3vid, dmVid, next) => delAndAssertResult(s3, { bucket, key, - versionId: dmVid, resultType: deleteDeleteMarker }, - err => next(err, s3vid)), - // should be able to get object originally put from s3 - (s3vid, next) => getAndAssertResult(s3, { bucket, key, - body: someBody, expectedVersionId: s3vid }, next), - // latest version in aws should now be object originally put - next => awsGetLatestVerId(key, someBody, next), - ], done); - }); - - it('multiple delete markers: should be able to get pre-existing ' + - 'versions after creating and deleting several delete markers', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => waitForVersioningBeforePut(s3, bucket, next), - next => putVersionsToAws(s3, bucket, key, [someBody], - (err, versionIds) => next(err, versionIds[0])), - (s3vid, next) => _createDeleteMarkers(s3, bucket, key, 3, - (err, dmVids) => next(err, s3vid, dmVids)), - (s3vid, dmVids, next) => _deleteDeleteMarkers(s3, bucket, key, - dmVids, () => next(null, s3vid)), - // should be able to get object originally put from s3 - (s3vid, next) => getAndAssertResult(s3, { bucket, key, - body: someBody, expectedVersionId: s3vid }, next), - // latest version in aws should now be object originally put - next => awsGetLatestVerId(key, someBody, next), - ], done); - }); - - it('multiple delete markers: should get NoSuchObject if only ' + - 'one of the delete markers is deleted', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => putVersionsToAws(s3, bucket, key, [someBody], - err => next(err)), - next => _createDeleteMarkers(s3, bucket, key, 3, - (err, dmVids) => next(err, dmVids[2])), - (lastDmVid, next) => delAndAssertResult(s3, { bucket, - key, versionId: lastDmVid, resultType: deleteDeleteMarker }, - err => next(err)), - next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' }, - next), - next => _awsGetAssertDeleted({ key, errorCode: 'NoSuchKey' }, - next), - ], done); - }); - - it('should get the new latest version after deleting the latest' + - 'specific version', done => { - const key = `somekey-${genUniqID()}`; - const data = [...Array(4).keys()].map(i => i.toString()); - async.waterfall([ - // put 3 null versions - next => mapToAwsPuts(s3, bucket, key, data.slice(0, 3), - err => next(err)), - // put one version - next => putVersionsToAws(s3, bucket, key, [data[3]], - (err, versionIds) => next(err, versionIds[0])), - // delete the latest version - (versionId, next) => delAndAssertResult(s3, { bucket, - key, versionId, resultType: deleteVersion }, - err => next(err)), - // should get the last null version - next => getAndAssertResult(s3, { bucket, key, - body: data[2], expectedVersionId: 'null' }, next), - next => awsGetLatestVerId(key, data[2], - err => next(err)), - // delete the null version - next => delAndAssertResult(s3, { bucket, - key, versionId: 'null', resultType: deleteVersion }, - err => next(err)), - // s3 metadata should report no existing versions for keyname - next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' }, - next), - // NOTE: latest version in aws will be the second null version - next => awsGetLatestVerId(key, data[1], - err => next(err)), - ], done); - }); + ); + + it('versioning enabled: should delete a delete marker in s3 and ' + 'aws successfully', done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => waitForVersioningBeforePut(s3, bucket, next), + next => + putVersionsToAws(s3, bucket, key, [someBody], (err, versionIds) => + next(err, versionIds[0]), + ), + // create a delete marker + (s3vid, next) => + delAndAssertResult(s3, { bucket, key, resultType: newDeleteMarker }, (err, delMarkerVid) => + next(err, s3vid, delMarkerVid), + ), + // delete delete marker + (s3vid, dmVid, next) => + delAndAssertResult( + s3, + { bucket, key, versionId: dmVid, resultType: deleteDeleteMarker }, + err => next(err, s3vid), + ), + // should be able to get object originally put from s3 + (s3vid, next) => + getAndAssertResult(s3, { bucket, key, body: someBody, expectedVersionId: s3vid }, next), + // latest version in aws should now be object originally put + next => awsGetLatestVerId(key, someBody, next), + ], + done, + ); + }); - it('should delete the correct version even if other versions or ' + - 'delete markers put directly on aws', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => putVersionsToAws(s3, bucket, key, [someBody], - (err, versionIds) => next(err, versionIds[0])), - (s3vid, next) => awsGetLatestVerId(key, someBody, - (err, awsVid) => next(err, s3vid, awsVid)), - // put an object in AWS - (s3vid, awsVid, next) => awsS3.send(new PutObjectCommand({ - Bucket: awsBucket, - Key: key - })).then(() => next(null, s3vid, awsVid)) - .catch(err => next(err)), - // create a delete marker in AWS - (s3vid, awsVid, next) => awsS3.send(new DeleteObjectCommand({ - Bucket: awsBucket, - Key: key - })).then(() => next(null, s3vid, awsVid)) - .catch(err => next(err)), - // delete original version in s3 - (s3vid, awsVid, next) => delAndAssertResult(s3, { bucket, key, - versionId: s3vid, resultType: deleteVersion }, - err => next(err, awsVid)), - (awsVid, next) => _getAssertDeleted(s3, { key, - errorCode: 'NoSuchKey' }, () => next(null, awsVid)), - (awsVerId, next) => { - _awsGetAssertDeleted({ key, - versionId: awsVerId, errorCode: 'NoSuchVersion' }, next); + it( + 'multiple delete markers: should be able to get pre-existing ' + + 'versions after creating and deleting several delete markers', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => waitForVersioningBeforePut(s3, bucket, next), + next => + putVersionsToAws(s3, bucket, key, [someBody], (err, versionIds) => + next(err, versionIds[0]), + ), + (s3vid, next) => + _createDeleteMarkers(s3, bucket, key, 3, (err, dmVids) => next(err, s3vid, dmVids)), + (s3vid, dmVids, next) => + _deleteDeleteMarkers(s3, bucket, key, dmVids, () => next(null, s3vid)), + // should be able to get object originally put from s3 + (s3vid, next) => + getAndAssertResult(s3, { bucket, key, body: someBody, expectedVersionId: s3vid }, next), + // latest version in aws should now be object originally put + next => awsGetLatestVerId(key, someBody, next), + ], + done, + ); }, - ], done); - }); + ); + + it( + 'multiple delete markers: should get NoSuchObject if only ' + 'one of the delete markers is deleted', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => putVersionsToAws(s3, bucket, key, [someBody], err => next(err)), + next => _createDeleteMarkers(s3, bucket, key, 3, (err, dmVids) => next(err, dmVids[2])), + (lastDmVid, next) => + delAndAssertResult( + s3, + { bucket, key, versionId: lastDmVid, resultType: deleteDeleteMarker }, + err => next(err), + ), + next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' }, next), + next => _awsGetAssertDeleted({ key, errorCode: 'NoSuchKey' }, next), + ], + done, + ); + }, + ); + + it('should get the new latest version after deleting the latest' + 'specific version', done => { + const key = `somekey-${genUniqID()}`; + const data = [...Array(4).keys()].map(i => i.toString()); + async.waterfall( + [ + // put 3 null versions + next => mapToAwsPuts(s3, bucket, key, data.slice(0, 3), err => next(err)), + // put one version + next => + putVersionsToAws(s3, bucket, key, [data[3]], (err, versionIds) => next(err, versionIds[0])), + // delete the latest version + (versionId, next) => + delAndAssertResult(s3, { bucket, key, versionId, resultType: deleteVersion }, err => + next(err), + ), + // should get the last null version + next => getAndAssertResult(s3, { bucket, key, body: data[2], expectedVersionId: 'null' }, next), + next => awsGetLatestVerId(key, data[2], err => next(err)), + // delete the null version + next => + delAndAssertResult(s3, { bucket, key, versionId: 'null', resultType: deleteVersion }, err => + next(err), + ), + // s3 metadata should report no existing versions for keyname + next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' }, next), + // NOTE: latest version in aws will be the second null version + next => awsGetLatestVerId(key, data[1], err => next(err)), + ], + done, + ); + }); - it('should not return an error deleting a version that was already ' + - 'deleted directly from AWS backend', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => putVersionsToAws(s3, bucket, key, [someBody], - (err, versionIds) => next(err, versionIds[0])), - (s3vid, next) => awsGetLatestVerId(key, someBody, - (err, awsVid) => next(err, s3vid, awsVid)), - // delete the object in AWS - (s3vid, awsVid, next) => awsS3.send(new DeleteObjectCommand({ - Bucket: awsBucket, - Key: key, - VersionId: awsVid - })).then(() => next(null, s3vid)) - .catch(err => next(err)), - // then try to delete in S3 - (s3vid, next) => delAndAssertResult(s3, { bucket, key, - versionId: s3vid, resultType: deleteVersion }, - err => next(err)), - next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' }, - next), - ], done); + it( + 'should delete the correct version even if other versions or ' + 'delete markers put directly on aws', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => + putVersionsToAws(s3, bucket, key, [someBody], (err, versionIds) => + next(err, versionIds[0]), + ), + (s3vid, next) => + awsGetLatestVerId(key, someBody, (err, awsVid) => next(err, s3vid, awsVid)), + // put an object in AWS + (s3vid, awsVid, next) => + awsS3 + .send( + new PutObjectCommand({ + Bucket: awsBucket, + Key: key, + }), + ) + .then(() => next(null, s3vid, awsVid)) + .catch(err => next(err)), + // create a delete marker in AWS + (s3vid, awsVid, next) => + awsS3 + .send( + new DeleteObjectCommand({ + Bucket: awsBucket, + Key: key, + }), + ) + .then(() => next(null, s3vid, awsVid)) + .catch(err => next(err)), + // delete original version in s3 + (s3vid, awsVid, next) => + delAndAssertResult( + s3, + { bucket, key, versionId: s3vid, resultType: deleteVersion }, + err => next(err, awsVid), + ), + (awsVid, next) => + _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' }, () => next(null, awsVid)), + (awsVerId, next) => { + _awsGetAssertDeleted({ key, versionId: awsVerId, errorCode: 'NoSuchVersion' }, next); + }, + ], + done, + ); + }, + ); + + it( + 'should not return an error deleting a version that was already ' + 'deleted directly from AWS backend', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => + putVersionsToAws(s3, bucket, key, [someBody], (err, versionIds) => + next(err, versionIds[0]), + ), + (s3vid, next) => + awsGetLatestVerId(key, someBody, (err, awsVid) => next(err, s3vid, awsVid)), + // delete the object in AWS + (s3vid, awsVid, next) => + awsS3 + .send( + new DeleteObjectCommand({ + Bucket: awsBucket, + Key: key, + VersionId: awsVid, + }), + ) + .then(() => next(null, s3vid)) + .catch(err => next(err)), + // then try to delete in S3 + (s3vid, next) => + delAndAssertResult( + s3, + { bucket, key, versionId: s3vid, resultType: deleteVersion }, + err => next(err), + ), + next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' }, next), + ], + done, + ); + }, + ); }); - }); -}); - -describeSkipIfNotMultiple('AWS backend delete object w. versioning: ' + - 'using bucket location constraint', function testSuite() { - this.timeout(120000); - const createBucketParams = { - Bucket: bucket, - CreateBucketConfiguration: { - LocationConstraint: awsLocation, - }, - }; - withV4(sigCfg => { - let bucketUtil; - let s3; - beforeEach(() => { - process.stdout.write('Creating bucket\n'); - bucketUtil = new BucketUtility('default', sigCfg); - s3 = bucketUtil.s3; - return s3.send(new CreateBucketCommand(createBucketParams)) - .catch(err => { - process.stdout.write(`Error creating bucket: ${err}\n`); - throw err; + }, +); + +describeSkipIfNotMultiple( + 'AWS backend delete object w. versioning: ' + 'using bucket location constraint', + function testSuite() { + this.timeout(120000); + const createBucketParams = { + Bucket: bucket, + CreateBucketConfiguration: { + LocationConstraint: awsLocation, + }, + }; + withV4(sigCfg => { + let bucketUtil; + let s3; + beforeEach(() => { + process.stdout.write('Creating bucket\n'); + bucketUtil = new BucketUtility('default', sigCfg); + s3 = bucketUtil.s3; + return s3.send(new CreateBucketCommand(createBucketParams)).catch(err => { + process.stdout.write(`Error creating bucket: ${err}\n`); + throw err; + }); }); - }); - afterEach(() => { - process.stdout.write('Emptying bucket\n'); - return bucketUtil.empty(bucket) - .then(() => { - process.stdout.write('Deleting bucket\n'); - return bucketUtil.deleteOne(bucket); - }) - .catch(err => { - process.stdout.write('Error emptying/deleting bucket: ' + - `${err}\n`); - throw err; + afterEach(() => { + process.stdout.write('Emptying bucket\n'); + return bucketUtil + .empty(bucket) + .then(() => { + process.stdout.write('Deleting bucket\n'); + return bucketUtil.deleteOne(bucket); + }) + .catch(err => { + process.stdout.write('Error emptying/deleting bucket: ' + `${err}\n`); + throw err; + }); }); - }); - it('versioning not configured: deleting non-existing object should ' + - 'not return version id or x-amz-delete-marker: true nor create a ' + - 'delete marker in aws ', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => delAndAssertResult(s3, { bucket, key, - resultType: nonVersionedDelete }, err => next(err)), - next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' }, - next), - next => _awsGetAssertDeleted({ key, errorCode: 'NoSuchKey' }, - next), - ], done); - }); - - it('versioning suspended: should create a delete marker in s3 ' + - 'and aws successfully when deleting non-existing object', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => suspendVersioning(s3, bucket, next), - next => delAndAssertResult(s3, { bucket, key, resultType: - newDeleteMarker }, err => next(err)), - next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' }, - next), - next => _awsGetAssertDeleted({ key, errorCode: 'NoSuchKey' }, - next), - ], done); - }); - - it('versioning enabled: should create a delete marker in s3 and ' + - 'aws successfully when deleting non-existing object', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => enableVersioning(s3, bucket, next), - next => delAndAssertResult(s3, { bucket, key, resultType: - newDeleteMarker }, err => next(err)), - next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' }, - next), - next => _awsGetAssertDeleted({ key, errorCode: 'NoSuchKey' }, - next), - ], done); + it( + 'versioning not configured: deleting non-existing object should ' + + 'not return version id or x-amz-delete-marker: true nor create a ' + + 'delete marker in aws ', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => + delAndAssertResult(s3, { bucket, key, resultType: nonVersionedDelete }, err => + next(err), + ), + next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' }, next), + next => _awsGetAssertDeleted({ key, errorCode: 'NoSuchKey' }, next), + ], + done, + ); + }, + ); + + it( + 'versioning suspended: should create a delete marker in s3 ' + + 'and aws successfully when deleting non-existing object', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => suspendVersioning(s3, bucket, next), + next => + delAndAssertResult(s3, { bucket, key, resultType: newDeleteMarker }, err => next(err)), + next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' }, next), + next => _awsGetAssertDeleted({ key, errorCode: 'NoSuchKey' }, next), + ], + done, + ); + }, + ); + + it( + 'versioning enabled: should create a delete marker in s3 and ' + + 'aws successfully when deleting non-existing object', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => enableVersioning(s3, bucket, next), + next => + delAndAssertResult(s3, { bucket, key, resultType: newDeleteMarker }, err => next(err)), + next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' }, next), + next => _awsGetAssertDeleted({ key, errorCode: 'NoSuchKey' }, next), + ], + done, + ); + }, + ); }); - }); -}); - - -describeSkipIfNotMultiple('AWS backend delete multiple objects w. versioning: ' + - 'using object location constraint', function testSuite() { - this.timeout(120000); - withV4(sigCfg => { - let bucketUtil; - let s3; - beforeEach(() => { - process.stdout.write('Creating bucket\n'); - bucketUtil = new BucketUtility('default', sigCfg); - s3 = bucketUtil.s3; - return s3.send(new CreateBucketCommand({ Bucket: bucket })) - .catch(err => { - process.stdout.write(`Error creating bucket: ${err}\n`); - throw err; + }, +); + +describeSkipIfNotMultiple( + 'AWS backend delete multiple objects w. versioning: ' + 'using object location constraint', + function testSuite() { + this.timeout(120000); + withV4(sigCfg => { + let bucketUtil; + let s3; + beforeEach(() => { + process.stdout.write('Creating bucket\n'); + bucketUtil = new BucketUtility('default', sigCfg); + s3 = bucketUtil.s3; + return s3.send(new CreateBucketCommand({ Bucket: bucket })).catch(err => { + process.stdout.write(`Error creating bucket: ${err}\n`); + throw err; + }); }); - }); - afterEach(() => { - process.stdout.write('Emptying bucket\n'); - return bucketUtil.empty(bucket) - .then(() => { - process.stdout.write('Deleting bucket\n'); - return bucketUtil.deleteOne(bucket); - }) - .catch(err => { - process.stdout.write('Error emptying/deleting bucket: ' + - `${err}\n`); - throw err; + afterEach(() => { + process.stdout.write('Emptying bucket\n'); + return bucketUtil + .empty(bucket) + .then(() => { + process.stdout.write('Deleting bucket\n'); + return bucketUtil.deleteOne(bucket); + }) + .catch(err => { + process.stdout.write('Error emptying/deleting bucket: ' + `${err}\n`); + throw err; + }); }); - }); - it('versioning not configured: if specifying "null" version, should ' + - 'delete specific version in AWS backend', done => { - const key = `somekey-${Date.now()}`; - async.waterfall([ - next => putToAwsBackend(s3, bucket, key, someBody, - err => next(err)), - next => awsGetLatestVerId(key, someBody, next), - (awsVerId, next) => delObjectsAndAssertResult(s3, { bucket, - key, versionId: 'null', resultType: deleteVersion }, - err => next(err, awsVerId)), - (awsVerId, next) => { - _awsGetAssertDeleted({ key, - versionId: awsVerId, errorCode: 'NoSuchVersion' }, next); + it( + 'versioning not configured: if specifying "null" version, should ' + + 'delete specific version in AWS backend', + done => { + const key = `somekey-${Date.now()}`; + async.waterfall( + [ + next => putToAwsBackend(s3, bucket, key, someBody, err => next(err)), + next => awsGetLatestVerId(key, someBody, next), + (awsVerId, next) => + delObjectsAndAssertResult( + s3, + { bucket, key, versionId: 'null', resultType: deleteVersion }, + err => next(err, awsVerId), + ), + (awsVerId, next) => { + _awsGetAssertDeleted({ key, versionId: awsVerId, errorCode: 'NoSuchVersion' }, next); + }, + ], + done, + ); }, - ], done); - }); - - it('versioning suspended: should delete a specific version in AWS ' + - 'backend successfully', done => { - const key = `somekey-${Date.now()}`; - async.waterfall([ - next => putNullVersionsToAws(s3, bucket, key, [someBody], - err => next(err)), - next => awsGetLatestVerId(key, someBody, next), - (awsVerId, next) => delObjectsAndAssertResult(s3, { bucket, - key, versionId: 'null', resultType: deleteVersion }, - err => next(err, awsVerId)), - (awsVerId, next) => { - _awsGetAssertDeleted({ key, - versionId: awsVerId, errorCode: 'NoSuchVersion' }, next); - }, - ], done); - }); + ); + + it('versioning suspended: should delete a specific version in AWS ' + 'backend successfully', done => { + const key = `somekey-${Date.now()}`; + async.waterfall( + [ + next => putNullVersionsToAws(s3, bucket, key, [someBody], err => next(err)), + next => awsGetLatestVerId(key, someBody, next), + (awsVerId, next) => + delObjectsAndAssertResult( + s3, + { bucket, key, versionId: 'null', resultType: deleteVersion }, + err => next(err, awsVerId), + ), + (awsVerId, next) => { + _awsGetAssertDeleted({ key, versionId: awsVerId, errorCode: 'NoSuchVersion' }, next); + }, + ], + done, + ); + }); - it('versioning enabled: should delete a specific version in AWS ' + - 'backend successfully', done => { - const key = `somekey-${Date.now()}`; - async.waterfall([ - next => waitForVersioningBeforePut(s3, bucket, next), - next => putVersionsToAws(s3, bucket, key, [someBody], - (err, versionIds) => next(err, versionIds[0])), - (s3vid, next) => awsGetLatestVerId(key, someBody, - (err, awsVid) => next(err, s3vid, awsVid)), - (s3VerId, awsVerId, next) => delObjectsAndAssertResult(s3, { bucket, - key, versionId: s3VerId, resultType: deleteVersion }, - err => next(err, awsVerId)), - (awsVerId, next) => { - _awsGetAssertDeleted({ key, - versionId: awsVerId, errorCode: 'NoSuchVersion' }, next); - }, - ], done); + it('versioning enabled: should delete a specific version in AWS ' + 'backend successfully', done => { + const key = `somekey-${Date.now()}`; + async.waterfall( + [ + next => waitForVersioningBeforePut(s3, bucket, next), + next => + putVersionsToAws(s3, bucket, key, [someBody], (err, versionIds) => + next(err, versionIds[0]), + ), + (s3vid, next) => awsGetLatestVerId(key, someBody, (err, awsVid) => next(err, s3vid, awsVid)), + (s3VerId, awsVerId, next) => + delObjectsAndAssertResult( + s3, + { bucket, key, versionId: s3VerId, resultType: deleteVersion }, + err => next(err, awsVerId), + ), + (awsVerId, next) => { + _awsGetAssertDeleted({ key, versionId: awsVerId, errorCode: 'NoSuchVersion' }, next); + }, + ], + done, + ); + }); }); - }); -}); + }, +); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/delete/deleteAzure.js b/tests/functional/aws-node-sdk/test/multipleBackend/delete/deleteAzure.js index 780f8fd632..eb17789564 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/delete/deleteAzure.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/delete/deleteAzure.js @@ -1,11 +1,13 @@ const assert = require('assert'); const async = require('async'); -const { CreateBucketCommand, +const { + CreateBucketCommand, PutObjectCommand, DeleteObjectCommand, GetObjectCommand, CreateMultipartUploadCommand, - AbortMultipartUploadCommand } = require('@aws-sdk/client-s3'); + AbortMultipartUploadCommand, +} = require('@aws-sdk/client-s3'); const BucketUtility = require('../../../lib/utility/bucket-util'); const withV4 = require('../../support/withV4'); const { @@ -26,12 +28,11 @@ const azureClient = getAzureClient(); const normalBody = Buffer.from('I am a body', 'utf8'); const azureTimeout = 20000; -const nonExistingId = process.env.AWS_ON_AIR ? - 'MhhyTHhmZ4cxSi4Y9SMe5P7UJAz7HLJ9' : - '3939393939393939393936493939393939393939756e6437'; +const nonExistingId = process.env.AWS_ON_AIR + ? 'MhhyTHhmZ4cxSi4Y9SMe5P7UJAz7HLJ9' + : '3939393939393939393936493939393939393939756e6437'; -describeSkipIfNotMultiple('Multiple backend delete object from Azure', -function testSuite() { +describeSkipIfNotMultiple('Multiple backend delete object from Azure', function testSuite() { this.timeout(250000); withV4(sigCfg => { let bucketUtil; @@ -41,8 +42,7 @@ function testSuite() { process.stdout.write('Creating bucket'); bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; - return s3.send(new CreateBucketCommand({ Bucket: azureContainerName })) - .catch(err => { + return s3.send(new CreateBucketCommand({ Bucket: azureContainerName })).catch(err => { process.stdout.write(`Error creating bucket: ${err}\n`); throw err; }); @@ -50,171 +50,223 @@ function testSuite() { after(() => { process.stdout.write('Emptying bucket\n'); - return bucketUtil.empty(azureContainerName) - .then(() => { - process.stdout.write('Deleting bucket\n'); - return bucketUtil.deleteOne(azureContainerName); - }) - .catch(err => { - process.stdout.write('Error emptying/deleting bucket: ' + - `${err}\n`); - throw err; - }); + return bucketUtil + .empty(azureContainerName) + .then(() => { + process.stdout.write('Deleting bucket\n'); + return bucketUtil.deleteOne(azureContainerName); + }) + .catch(err => { + process.stdout.write('Error emptying/deleting bucket: ' + `${err}\n`); + throw err; + }); }); keys.forEach(key => { const keyName = uniqName(keyObject); describe(`${key.describe} size`, () => { before(done => { - s3.send(new PutObjectCommand({ - Bucket: azureContainerName, - Key: keyName, - Body: key.body, - Metadata: { - 'scal-location-constraint': azureLocation, - }, - })).then(() => done()); + s3.send( + new PutObjectCommand({ + Bucket: azureContainerName, + Key: keyName, + Body: key.body, + Metadata: { + 'scal-location-constraint': azureLocation, + }, + }), + ).then(() => done()); }); - it(`should delete an ${key.describe} object from Azure`, - done => { - s3.send(new DeleteObjectCommand({ - Bucket: azureContainerName, - Key: keyName, - })).then(() => { - setTimeout(() => azureClient.getContainerClient(azureContainerName) - .getProperties(keyName) - .then(() => assert.fail('Expected error'), err => { - assert.strictEqual(err.statusCode, 404); - assert.strictEqual(err.code, 'NotFound'); - return done(); - }), azureTimeout); - }).catch(err => { - assert.equal(err, null, 'Expected success ' + - `but got error ${err}`); - }); + it(`should delete an ${key.describe} object from Azure`, done => { + s3.send( + new DeleteObjectCommand({ + Bucket: azureContainerName, + Key: keyName, + }), + ) + .then(() => { + setTimeout( + () => + azureClient + .getContainerClient(azureContainerName) + .getProperties(keyName) + .then( + () => assert.fail('Expected error'), + err => { + assert.strictEqual(err.statusCode, 404); + assert.strictEqual(err.code, 'NotFound'); + return done(); + }, + ), + azureTimeout, + ); + }) + .catch(err => { + assert.equal(err, null, 'Expected success ' + `but got error ${err}`); + }); }); }); }); - describe('delete from Azure location with bucketMatch set to false', - () => { + describe('delete from Azure location with bucketMatch set to false', () => { beforeEach(function beforeF(done) { this.currentTest.azureObject = uniqName(keyObject); - s3.send(new PutObjectCommand({ - Bucket: azureContainerName, - Key: this.currentTest.azureObject, - Body: normalBody, - Metadata: { - 'scal-location-constraint': azureLocationMismatch, - }, - })).then(() => done()); + s3.send( + new PutObjectCommand({ + Bucket: azureContainerName, + Key: this.currentTest.azureObject, + Body: normalBody, + Metadata: { + 'scal-location-constraint': azureLocationMismatch, + }, + }), + ).then(() => done()); }); it('should delete object', function itF(done) { - s3.send(new DeleteObjectCommand({ - Bucket: azureContainerName, - Key: this.test.azureObject, - })).then(() => { - setTimeout(() => - azureClient.getContainerClient(azureContainerName) - .getProperties(`${azureContainerName}/${this.test.azureObject}`) - .then(() => assert.fail('Expected error'), err => { - assert.strictEqual(err.statusCode, 404); - assert.strictEqual(err.code, 'NotFound'); - return done(); - }), azureTimeout); - }).catch(err => { - assert.equal(err, null, 'Expected success ' + - `but got error ${err}`); - }); + s3.send( + new DeleteObjectCommand({ + Bucket: azureContainerName, + Key: this.test.azureObject, + }), + ) + .then(() => { + setTimeout( + () => + azureClient + .getContainerClient(azureContainerName) + .getProperties(`${azureContainerName}/${this.test.azureObject}`) + .then( + () => assert.fail('Expected error'), + err => { + assert.strictEqual(err.statusCode, 404); + assert.strictEqual(err.code, 'NotFound'); + return done(); + }, + ), + azureTimeout, + ); + }) + .catch(err => { + assert.equal(err, null, 'Expected success ' + `but got error ${err}`); + }); }); }); describe('returning no error', () => { beforeEach(function beF(done) { this.currentTest.azureObject = uniqName(keyObject); - s3.send(new PutObjectCommand({ - Bucket: azureContainerName, - Key: this.currentTest.azureObject, - Body: normalBody, - Metadata: { - 'scal-location-constraint': azureLocation, - }, - })).then(() => { - azureClient.getContainerClient(azureContainerName) - .deleteBlob(this.currentTest.azureObject).then(done, err => { - assert.equal(err, null, 'Expected success but got ' + - `error ${err}`); - done(err); - }); - }).catch(err => { - assert.equal(err, null, 'Expected success but got ' + - `error ${err}`); - done(); - }); + s3.send( + new PutObjectCommand({ + Bucket: azureContainerName, + Key: this.currentTest.azureObject, + Body: normalBody, + Metadata: { + 'scal-location-constraint': azureLocation, + }, + }), + ) + .then(() => { + azureClient + .getContainerClient(azureContainerName) + .deleteBlob(this.currentTest.azureObject) + .then(done, err => { + assert.equal(err, null, 'Expected success but got ' + `error ${err}`); + done(err); + }); + }) + .catch(err => { + assert.equal(err, null, 'Expected success but got ' + `error ${err}`); + done(); + }); }); - it('should return no error on deleting an object deleted ' + - 'from Azure', function itF(done) { - s3.send(new DeleteObjectCommand({ - Bucket: azureContainerName, - Key: this.test.azureObject, - })).then(() => { - done(); - }).catch(err => { - assert.equal(err, null, 'Expected success but got ' + - `error ${err}`); - done(); - }); + it('should return no error on deleting an object deleted ' + 'from Azure', function itF(done) { + s3.send( + new DeleteObjectCommand({ + Bucket: azureContainerName, + Key: this.test.azureObject, + }), + ) + .then(() => { + done(); + }) + .catch(err => { + assert.equal(err, null, 'Expected success but got ' + `error ${err}`); + done(); + }); }); }); describe('Versioning:: ', () => { beforeEach(function beF(done) { this.currentTest.azureObject = uniqName(keyObject); - s3.send(new PutObjectCommand({ - Bucket: azureContainerName, - Key: this.currentTest.azureObject, - Body: normalBody, - Metadata: { - 'scal-location-constraint': azureLocation, - }, - })).then(() => done()); - }); - - it('should not delete object when deleting a non-existing ' + - 'version from Azure', function itF(done) { - async.waterfall([ - next => s3.send(new DeleteObjectCommand({ - Bucket: azureContainerName, - Key: this.test.azureObject, - VersionId: nonExistingId, - })).then(() => next()) - .catch(err => { - next(err); - }), - next => s3.send(new GetObjectCommand({ + s3.send( + new PutObjectCommand({ Bucket: azureContainerName, - Key: this.test.azureObject, - })).then(res => { - assert.deepStrictEqual(res.Body, normalBody); - return next(); - }).catch(err => { - assert.equal(err, null, 'getObject: Expected success ' + - `but got error ${err}`); - next(err); - }), - next => azureClient.getContainerClient(azureContainerName) - .getBlobClient(this.test.azureObject) - .downloadToBuffer().then(res => { - assert.deepStrictEqual(res, normalBody); - return next(); - }, err => { - assert.equal(err, null, 'getBlobToText: Expected ' + - `successbut got error ${err}`); - return next(); + Key: this.currentTest.azureObject, + Body: normalBody, + Metadata: { + 'scal-location-constraint': azureLocation, + }, }), - ], done); + ).then(() => done()); + }); + + it('should not delete object when deleting a non-existing ' + 'version from Azure', function itF(done) { + async.waterfall( + [ + next => + s3 + .send( + new DeleteObjectCommand({ + Bucket: azureContainerName, + Key: this.test.azureObject, + VersionId: nonExistingId, + }), + ) + .then(() => next()) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new GetObjectCommand({ + Bucket: azureContainerName, + Key: this.test.azureObject, + }), + ) + .then(res => { + assert.deepStrictEqual(res.Body, normalBody); + return next(); + }) + .catch(err => { + assert.equal(err, null, 'getObject: Expected success ' + `but got error ${err}`); + next(err); + }), + next => + azureClient + .getContainerClient(azureContainerName) + .getBlobClient(this.test.azureObject) + .downloadToBuffer() + .then( + res => { + assert.deepStrictEqual(res, normalBody); + return next(); + }, + err => { + assert.equal( + err, + null, + 'getBlobToText: Expected ' + `successbut got error ${err}`, + ); + return next(); + }, + ), + ], + done, + ); }); }); @@ -227,50 +279,60 @@ function testSuite() { Body: normalBody, Metadata: { 'scal-location-constraint': azureLocation }, }; - s3.send(new PutObjectCommand(params)).then(() => { - const params = { - Bucket: azureContainerName, - Key: this.currentTest.key, - Metadata: { 'scal-location-constraint': azureLocation }, - }; - s3.send(new CreateMultipartUploadCommand(params)).then(res => { - this.currentTest.uploadId = res.UploadId; - setTimeout(() => done(), azureTimeout); - }).catch(err => { - assert.equal(err, null, 'Err initiating MPU on ' + - `Azure: ${err}`); + s3.send(new PutObjectCommand(params)) + .then(() => { + const params = { + Bucket: azureContainerName, + Key: this.currentTest.key, + Metadata: { 'scal-location-constraint': azureLocation }, + }; + s3.send(new CreateMultipartUploadCommand(params)) + .then(res => { + this.currentTest.uploadId = res.UploadId; + setTimeout(() => done(), azureTimeout); + }) + .catch(err => { + assert.equal(err, null, 'Err initiating MPU on ' + `Azure: ${err}`); + done(); + }); + }) + .catch(err => { + assert.equal(err, null, 'Err putting object to Azure: ' + `${err}`); done(); }); - }).catch(err => { - assert.equal(err, null, 'Err putting object to Azure: ' + - `${err}`); - done(); - }); }); afterEach(function afF(done) { - s3.send(new AbortMultipartUploadCommand({ - Bucket: azureContainerName, - Key: this.currentTest.key, - UploadId: this.currentTest.uploadId, - })).then(() => { - setTimeout(() => done(), azureTimeout); - }).catch(err => { - assert.equal(err, null, `Err aborting MPU: ${err}`); - done(); - }); + s3.send( + new AbortMultipartUploadCommand({ + Bucket: azureContainerName, + Key: this.currentTest.key, + UploadId: this.currentTest.uploadId, + }), + ) + .then(() => { + setTimeout(() => done(), azureTimeout); + }) + .catch(err => { + assert.equal(err, null, `Err aborting MPU: ${err}`); + done(); + }); }); it('should return InternalError', function itFn(done) { - s3.send(new DeleteObjectCommand({ - Bucket: azureContainerName, - Key: this.test.key, - })).then(() => { - done(); - }).catch(err => { - assert.strictEqual(err.code, 'MPUinProgress'); - done(); - }); + s3.send( + new DeleteObjectCommand({ + Bucket: azureContainerName, + Key: this.test.key, + }), + ) + .then(() => { + done(); + }) + .catch(err => { + assert.strictEqual(err.code, 'MPUinProgress'); + done(); + }); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/delete/deleteGcp.js b/tests/functional/aws-node-sdk/test/multipleBackend/delete/deleteGcp.js index 6bf5daf676..ea4773acbf 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/delete/deleteGcp.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/delete/deleteGcp.js @@ -1,16 +1,8 @@ const assert = require('assert'); -const { CreateBucketCommand, - PutObjectCommand, - DeleteObjectCommand, - GetObjectCommand } = require('@aws-sdk/client-s3'); +const { CreateBucketCommand, PutObjectCommand, DeleteObjectCommand, GetObjectCommand } = require('@aws-sdk/client-s3'); const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); -const { - gcpLocation, - gcpLocationMismatch, - genUniqID, - describeSkipIfNotMultiple, -} = require('../utils'); +const { gcpLocation, gcpLocationMismatch, genUniqID, describeSkipIfNotMultiple } = require('../utils'); const bucket = `deletegcp${genUniqID()}`; const gcpObject = `gcpObject-${genUniqID()}`; @@ -20,8 +12,7 @@ const mismatchObject = `mismatchObject-${genUniqID()}`; const body = Buffer.from('I am a body', 'utf8'); const bigBody = Buffer.alloc(10485760); -describeSkipIfNotMultiple('Multiple backend delete', -function testSuite() { +describeSkipIfNotMultiple('Multiple backend delete', function testSuite() { this.timeout(120000); withV4(sigCfg => { let bucketUtil; @@ -31,45 +22,59 @@ function testSuite() { process.stdout.write('Creating bucket\n'); bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; - return s3.send(new CreateBucketCommand({ Bucket: bucket })) - .catch(err => { - process.stdout.write(`Error creating bucket: ${err}\n`); - throw err; - }).then(() => { - process.stdout.write('Putting object to GCP\n'); - const params = { Bucket: bucket, Key: gcpObject, Body: body, - Metadata: { 'scal-location-constraint': gcpLocation } }; - return s3.send(new PutObjectCommand(params)); - }) - .then(() => { - process.stdout.write('Putting 0-byte object to GCP\n'); - const params = { Bucket: bucket, Key: emptyObject, - Metadata: { 'scal-location-constraint': gcpLocation } }; - return s3.send(new PutObjectCommand(params)); - }) - .then(() => { - process.stdout.write('Putting large object to GCP\n'); - const params = { Bucket: bucket, Key: bigObject, - Body: bigBody, - Metadata: { 'scal-location-constraint': gcpLocation } }; - return s3.send(new PutObjectCommand(params)); - }) - .then(() => { - process.stdout.write('Putting object to GCP\n'); - const params = { Bucket: bucket, Key: mismatchObject, - Body: body, Metadata: - { 'scal-location-constraint': gcpLocationMismatch } }; - return s3.send(new PutObjectCommand(params)); - }) - .catch(err => { - process.stdout.write(`Error putting objects: ${err}\n`); - throw err; - }); + return s3 + .send(new CreateBucketCommand({ Bucket: bucket })) + .catch(err => { + process.stdout.write(`Error creating bucket: ${err}\n`); + throw err; + }) + .then(() => { + process.stdout.write('Putting object to GCP\n'); + const params = { + Bucket: bucket, + Key: gcpObject, + Body: body, + Metadata: { 'scal-location-constraint': gcpLocation }, + }; + return s3.send(new PutObjectCommand(params)); + }) + .then(() => { + process.stdout.write('Putting 0-byte object to GCP\n'); + const params = { + Bucket: bucket, + Key: emptyObject, + Metadata: { 'scal-location-constraint': gcpLocation }, + }; + return s3.send(new PutObjectCommand(params)); + }) + .then(() => { + process.stdout.write('Putting large object to GCP\n'); + const params = { + Bucket: bucket, + Key: bigObject, + Body: bigBody, + Metadata: { 'scal-location-constraint': gcpLocation }, + }; + return s3.send(new PutObjectCommand(params)); + }) + .then(() => { + process.stdout.write('Putting object to GCP\n'); + const params = { + Bucket: bucket, + Key: mismatchObject, + Body: body, + Metadata: { 'scal-location-constraint': gcpLocationMismatch }, + }; + return s3.send(new PutObjectCommand(params)); + }) + .catch(err => { + process.stdout.write(`Error putting objects: ${err}\n`); + throw err; + }); }); after(() => { process.stdout.write('Deleting bucket\n'); - return bucketUtil.deleteOne(bucket) - .catch(err => { + return bucketUtil.deleteOne(bucket).catch(err => { process.stdout.write(`Error deleting bucket: ${err}\n`); throw err; }); @@ -78,42 +83,51 @@ function testSuite() { const deleteTests = [ { msg: 'should delete object from GCP', - Bucket: bucket, Key: gcpObject, + Bucket: bucket, + Key: gcpObject, }, { msg: 'should delete 0-byte object from GCP', - Bucket: bucket, Key: emptyObject, + Bucket: bucket, + Key: emptyObject, }, { msg: 'should delete large object from GCP', - Bucket: bucket, Key: bigObject, + Bucket: bucket, + Key: bigObject, }, { - msg: 'should delete object from GCP location with ' + - 'bucketMatch set to false', - Bucket: bucket, Key: mismatchObject, + msg: 'should delete object from GCP location with ' + 'bucketMatch set to false', + Bucket: bucket, + Key: mismatchObject, }, ]; deleteTests.forEach(test => { const { msg, Bucket, Key } = test; - it(msg, done => s3.send(new DeleteObjectCommand({ Bucket, Key })) - .then(() => s3.send(new GetObjectCommand({ Bucket, Key })) + it(msg, done => + s3.send(new DeleteObjectCommand({ Bucket, Key })).then(() => + s3 + .send(new GetObjectCommand({ Bucket, Key })) + .then(() => { + assert.fail('Expected error but got success'); + }) + .catch(err => { + assert.strictEqual(err.code, 'NoSuchKey', 'Expected ' + 'error but got success'); + return done(); + }), + ), + ); + }); + + it('should return success if the object does not exist', done => + s3 + .send(new DeleteObjectCommand({ Bucket: bucket, Key: 'noop' })) .then(() => { assert.fail('Expected error but got success'); - }).catch(err => { - assert.strictEqual(err.code, 'NoSuchKey', 'Expected ' + - 'error but got success'); + }) + .catch(err => { + assert.strictEqual(err, null, `Expected success, got error ${JSON.stringify(err)}`); return done(); - }))); - }); - - it('should return success if the object does not exist', - done => s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: 'noop' })).then(() => { - assert.fail('Expected error but got success'); - }).catch(err => { - assert.strictEqual(err, null, - `Expected success, got error ${JSON.stringify(err)}`); - return done(); - })); + })); }); }); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/get/get.js b/tests/functional/aws-node-sdk/test/multipleBackend/get/get.js index 086ae9ec33..9e1607268b 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/get/get.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/get/get.js @@ -43,8 +43,7 @@ describe('Multiple backend get object', function testSuite() { bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; const command = new CreateBucketCommand({ Bucket: bucket }); - return s3.send(command) - .catch(err => { + return s3.send(command).catch(err => { process.stdout.write(`Error creating bucket: ${err}\n`); throw err; }); @@ -52,102 +51,102 @@ describe('Multiple backend get object', function testSuite() { after(() => { process.stdout.write('Emptying bucket\n'); - return bucketUtil.empty(bucket) - .then(() => { - process.stdout.write('Deleting bucket\n'); - return bucketUtil.deleteOne(bucket); - }) - .catch(err => { - process.stdout.write('Error emptying/deleting bucket: ' + - `${err}\n`); - throw err; - }); + return bucketUtil + .empty(bucket) + .then(() => { + process.stdout.write('Deleting bucket\n'); + return bucketUtil.deleteOne(bucket); + }) + .catch(err => { + process.stdout.write('Error emptying/deleting bucket: ' + `${err}\n`); + throw err; + }); }); // aws-sdk now (v2.363.0) returns 'UriParameterError' error - it.skip('should return an error to get request without a valid ' + - 'bucket name', - done => { - const command = new GetObjectCommand({ Bucket: '', Key: 'somekey' }); - s3.send(command) - .then(() => done(new Error('Expected failure but got success'))) - .catch(err => { - assert.notEqual(err, null, - 'Expected failure but got success'); - assert.strictEqual(err.name, 'MethodNotAllowed'); - done(); - }); - }); - it('should return NoSuchKey error when no such object', - done => { - const command = new GetObjectCommand({ Bucket: bucket, Key: 'nope' }); - s3.send(command) - .then(() => done(new Error('Expected failure but got success'))) - .catch(err => { - assert.notEqual(err, null, - 'Expected failure but got success'); - assert.strictEqual(err.name, 'NoSuchKey'); - done(); - }); - }); + it.skip('should return an error to get request without a valid ' + 'bucket name', done => { + const command = new GetObjectCommand({ Bucket: '', Key: 'somekey' }); + s3.send(command) + .then(() => done(new Error('Expected failure but got success'))) + .catch(err => { + assert.notEqual(err, null, 'Expected failure but got success'); + assert.strictEqual(err.name, 'MethodNotAllowed'); + done(); + }); + }); + it('should return NoSuchKey error when no such object', done => { + const command = new GetObjectCommand({ Bucket: bucket, Key: 'nope' }); + s3.send(command) + .then(() => done(new Error('Expected failure but got success'))) + .catch(err => { + assert.notEqual(err, null, 'Expected failure but got success'); + assert.strictEqual(err.name, 'NoSuchKey'); + done(); + }); + }); - describeSkipIfNotMultiple('Complete MPU then get object on AWS ' + - 'location with bucketMatch: true ', () => { + describeSkipIfNotMultiple('Complete MPU then get object on AWS ' + 'location with bucketMatch: true ', () => { beforeEach(function beforeEachFn() { this.currentTest.key = `somekey-${genUniqID()}`; bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; - return s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: this.currentTest.key, - Metadata: { 'scal-location-constraint': awsLocation }, - })) - .then(res => { - const uploadId = res.UploadId; - const partBody = Buffer.from('helloworld', 'utf8'); - const uploadPartInput = { - Bucket: bucket, - Key: this.currentTest.key, - PartNumber: 1, - UploadId: uploadId, - Body: partBody, - ContentLength: partBody.length, - }; - const uploadPartCommand = new UploadPartCommand(uploadPartInput); - uploadPartCommand.middlewareStack.add(next => async args => { - const headers = args.request?.headers; - if (headers) { - headers['Content-Length'] = `${partBody.length}`; - headers['x-amz-decoded-content-length'] = `${partBody.length}`; - } - return next(args); - }, { step: 'build' }); - return s3.send(uploadPartCommand) - .then(partRes => { - const eTag = partRes.ETag; - return s3.send(new CompleteMultipartUploadCommand({ + return s3 + .send( + new CreateMultipartUploadCommand({ Bucket: bucket, Key: this.currentTest.key, - MultipartUpload: { - Parts: [ - { - ETag: eTag, - PartNumber: 1, - }, - ], - }, + Metadata: { 'scal-location-constraint': awsLocation }, + }), + ) + .then(res => { + const uploadId = res.UploadId; + const partBody = Buffer.from('helloworld', 'utf8'); + const uploadPartInput = { + Bucket: bucket, + Key: this.currentTest.key, + PartNumber: 1, UploadId: uploadId, - })); + Body: partBody, + ContentLength: partBody.length, + }; + const uploadPartCommand = new UploadPartCommand(uploadPartInput); + uploadPartCommand.middlewareStack.add( + next => async args => { + const headers = args.request?.headers; + if (headers) { + headers['Content-Length'] = `${partBody.length}`; + headers['x-amz-decoded-content-length'] = `${partBody.length}`; + } + return next(args); + }, + { step: 'build' }, + ); + return s3.send(uploadPartCommand).then(partRes => { + const eTag = partRes.ETag; + return s3.send( + new CompleteMultipartUploadCommand({ + Bucket: bucket, + Key: this.currentTest.key, + MultipartUpload: { + Parts: [ + { + ETag: eTag, + PartNumber: 1, + }, + ], + }, + UploadId: uploadId, + }), + ); + }); + }) + .catch(err => { + process.stdout.write(`Error in beforeEach: ${err}\n`); + throw err; }); - }) - .catch(err => { - process.stdout.write(`Error in beforeEach: ${err}\n`); - throw err; - }); }); - it('should get object from MPU on AWS ' + - 'location with bucketMatch: true ', function it(done) { + it('should get object from MPU on AWS ' + 'location with bucketMatch: true ', function it(done) { const command = new GetObjectCommand({ Bucket: bucket, Key: this.test.key, @@ -156,75 +155,78 @@ describe('Multiple backend get object', function testSuite() { .then(res => { assert.strictEqual(res.ContentLength, 10); assert.strictEqual(res.Body.toString(), 'helloworld'); - assert.deepStrictEqual(res.Metadata, - { 'scal-location-constraint': awsLocation }); + assert.deepStrictEqual(res.Metadata, { 'scal-location-constraint': awsLocation }); done(); }) .catch(err => { - assert.equal(err, null, 'Expected success but got ' + - `error ${err}`); + assert.equal(err, null, 'Expected success but got ' + `error ${err}`); done(err); }); }); }); - describeSkipIfNotMultiple('Complete MPU then get object on AWS ' + - 'location with bucketMatch: false ', () => { + describeSkipIfNotMultiple('Complete MPU then get object on AWS ' + 'location with bucketMatch: false ', () => { beforeEach(function beforeEachFn() { this.currentTest.key = `somekey-${genUniqID()}`; bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; - return s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: this.currentTest.key, - Metadata: { 'scal-location-constraint': awsLocationMismatch }, - })) - .then(res => { - const uploadId = res.UploadId; - const partBody = Buffer.from('helloworld', 'utf8'); - const uploadPartInput = { - Bucket: bucket, - Key: this.currentTest.key, - PartNumber: 1, - UploadId: uploadId, - Body: partBody, - ContentLength: partBody.length, - }; - const uploadPartCommand = new UploadPartCommand(uploadPartInput); - uploadPartCommand.middlewareStack.add(next => async args => { - const headers = args.request?.headers; - if (headers) { - headers['content-length'] = `${partBody.length}`; - headers['x-amz-decoded-content-length'] = `${partBody.length}`; - } - return next(args); - }, { step: 'build' }); - return s3.send(uploadPartCommand) - .then(partRes => { - const eTag = partRes.ETag; - return s3.send(new CompleteMultipartUploadCommand({ + return s3 + .send( + new CreateMultipartUploadCommand({ Bucket: bucket, Key: this.currentTest.key, - MultipartUpload: { - Parts: [ - { - ETag: eTag, - PartNumber: 1, - }, - ], - }, + Metadata: { 'scal-location-constraint': awsLocationMismatch }, + }), + ) + .then(res => { + const uploadId = res.UploadId; + const partBody = Buffer.from('helloworld', 'utf8'); + const uploadPartInput = { + Bucket: bucket, + Key: this.currentTest.key, + PartNumber: 1, UploadId: uploadId, - })); + Body: partBody, + ContentLength: partBody.length, + }; + const uploadPartCommand = new UploadPartCommand(uploadPartInput); + uploadPartCommand.middlewareStack.add( + next => async args => { + const headers = args.request?.headers; + if (headers) { + headers['content-length'] = `${partBody.length}`; + headers['x-amz-decoded-content-length'] = `${partBody.length}`; + } + return next(args); + }, + { step: 'build' }, + ); + return s3.send(uploadPartCommand).then(partRes => { + const eTag = partRes.ETag; + return s3.send( + new CompleteMultipartUploadCommand({ + Bucket: bucket, + Key: this.currentTest.key, + MultipartUpload: { + Parts: [ + { + ETag: eTag, + PartNumber: 1, + }, + ], + }, + UploadId: uploadId, + }), + ); + }); + }) + .catch(err => { + process.stdout.write(`Error in beforeEach: ${err}\n`); + throw err; }); - }) - .catch(err => { - process.stdout.write(`Error in beforeEach: ${err}\n`); - throw err; - }); }); - it('should get object from MPU on AWS ' + - 'location with bucketMatch: false ', function it(done) { + it('should get object from MPU on AWS ' + 'location with bucketMatch: false ', function it(done) { const command = new GetObjectCommand({ Bucket: bucket, Key: this.test.key, @@ -233,81 +235,79 @@ describe('Multiple backend get object', function testSuite() { .then(res => { assert.strictEqual(res.ContentLength, 10); assert.strictEqual(res.Body.toString(), 'helloworld'); - assert.deepStrictEqual(res.Metadata, - { 'scal-location-constraint': awsLocationMismatch }); + assert.deepStrictEqual(res.Metadata, { 'scal-location-constraint': awsLocationMismatch }); done(); }) .catch(err => { - assert.equal(err, null, 'Expected success but got ' + - `error ${err}`); + assert.equal(err, null, 'Expected success but got ' + `error ${err}`); done(err); }); }); }); - describeSkipIfNotMultiple('with objects in all available backends ' + - '(mem/file/AWS)', () => { + describeSkipIfNotMultiple('with objects in all available backends ' + '(mem/file/AWS)', () => { before(() => { process.stdout.write('Putting object to mem\n'); - const memCommand = new PutObjectCommand({ - Bucket: bucket, + const memCommand = new PutObjectCommand({ + Bucket: bucket, Key: memObject, Body: body, Metadata: { 'scal-location-constraint': memLocation }, }); - return s3.send(memCommand) - .then(() => { - process.stdout.write('Putting object to file\n'); - const fileCommand = new PutObjectCommand({ - Bucket: bucket, - Key: fileObject, - Body: body, - Metadata: { 'scal-location-constraint': fileLocation }, - }); - return s3.send(fileCommand); - }) - .then(() => { - process.stdout.write('Putting object to AWS\n'); - const awsCommand = new PutObjectCommand({ - Bucket: bucket, - Key: awsObject, - Body: body, - Metadata: { 'scal-location-constraint': awsLocation }, - }); - return s3.send(awsCommand); - }) - .then(() => { - process.stdout.write('Putting 0-byte object to mem\n'); - const emptyCommand = new PutObjectCommand({ - Bucket: bucket, - Key: emptyObject, - Metadata: { 'scal-location-constraint': memLocation }, - }); - return s3.send(emptyCommand); - }) - .then(() => { - process.stdout.write('Putting 0-byte object to AWS\n'); - const emptyAwsCommand = new PutObjectCommand({ - Bucket: bucket, - Key: emptyAwsObject, - Metadata: { 'scal-location-constraint': awsLocation }, - }); - return s3.send(emptyAwsCommand); - }) - .then(() => { - process.stdout.write('Putting large object to AWS\n'); - const bigCommand = new PutObjectCommand({ - Bucket: bucket, - Key: bigObject, - Body: bigBody, - Metadata: { 'scal-location-constraint': awsLocation }, + return s3 + .send(memCommand) + .then(() => { + process.stdout.write('Putting object to file\n'); + const fileCommand = new PutObjectCommand({ + Bucket: bucket, + Key: fileObject, + Body: body, + Metadata: { 'scal-location-constraint': fileLocation }, + }); + return s3.send(fileCommand); + }) + .then(() => { + process.stdout.write('Putting object to AWS\n'); + const awsCommand = new PutObjectCommand({ + Bucket: bucket, + Key: awsObject, + Body: body, + Metadata: { 'scal-location-constraint': awsLocation }, + }); + return s3.send(awsCommand); + }) + .then(() => { + process.stdout.write('Putting 0-byte object to mem\n'); + const emptyCommand = new PutObjectCommand({ + Bucket: bucket, + Key: emptyObject, + Metadata: { 'scal-location-constraint': memLocation }, + }); + return s3.send(emptyCommand); + }) + .then(() => { + process.stdout.write('Putting 0-byte object to AWS\n'); + const emptyAwsCommand = new PutObjectCommand({ + Bucket: bucket, + Key: emptyAwsObject, + Metadata: { 'scal-location-constraint': awsLocation }, + }); + return s3.send(emptyAwsCommand); + }) + .then(() => { + process.stdout.write('Putting large object to AWS\n'); + const bigCommand = new PutObjectCommand({ + Bucket: bucket, + Key: bigObject, + Body: bigBody, + Metadata: { 'scal-location-constraint': awsLocation }, + }); + return s3.send(bigCommand); + }) + .catch(err => { + process.stdout.write(`Error putting objects: ${err}\n`); + throw err; }); - return s3.send(bigCommand); - }) - .catch(err => { - process.stdout.write(`Error putting objects: ${err}\n`); - throw err; - }); }); it('should get an object from mem', done => { const command = new GetObjectCommand({ Bucket: bucket, Key: memObject }); @@ -317,8 +317,7 @@ describe('Multiple backend get object', function testSuite() { done(); }) .catch(err => { - assert.equal(err, null, 'Expected success but got ' + - `error ${err}`); + assert.equal(err, null, 'Expected success but got ' + `error ${err}`); done(err); }); }); @@ -330,8 +329,7 @@ describe('Multiple backend get object', function testSuite() { done(); }) .catch(err => { - assert.equal(err, null, 'Expected success but got ' + - `error ${err}`); + assert.equal(err, null, 'Expected success but got ' + `error ${err}`); done(err); }); }); @@ -343,8 +341,7 @@ describe('Multiple backend get object', function testSuite() { done(); }) .catch(err => { - assert.equal(err, null, 'Expected success but got error ' + - `error ${err}`); + assert.equal(err, null, 'Expected success but got error ' + `error ${err}`); done(err); }); }); @@ -356,8 +353,7 @@ describe('Multiple backend get object', function testSuite() { done(); }) .catch(err => { - assert.equal(err, null, 'Expected success but got ' + - `error ${err}`); + assert.equal(err, null, 'Expected success but got ' + `error ${err}`); done(err); }); }); @@ -369,8 +365,7 @@ describe('Multiple backend get object', function testSuite() { done(); }) .catch(err => { - assert.equal(err, null, 'Expected success but got ' + - `error ${err}`); + assert.equal(err, null, 'Expected success but got ' + `error ${err}`); done(err); }); }); @@ -382,39 +377,36 @@ describe('Multiple backend get object', function testSuite() { done(); }) .catch(err => { - assert.equal(err, null, 'Expected success but got ' + - `error ${err}`); + assert.equal(err, null, 'Expected success but got ' + `error ${err}`); done(err); }); }); it('should get an object using range query from AWS', done => { - const command = new GetObjectCommand({ - Bucket: bucket, + const command = new GetObjectCommand({ + Bucket: bucket, Key: bigObject, - Range: 'bytes=0-9' + Range: 'bytes=0-9', }); s3.send(command) .then(res => { assert.strictEqual(res.ContentLength, 10); - assert.strictEqual(res.ContentRange, - `bytes 0-9/${bigBodyLen}`); + assert.strictEqual(res.ContentRange, `bytes 0-9/${bigBodyLen}`); assert.strictEqual(res.ETag, `"${bigMD5}"`); done(); }) .catch(err => { - assert.equal(err, null, 'Expected success but got ' + - `error ${err}`); + assert.equal(err, null, 'Expected success but got ' + `error ${err}`); done(err); }); }); }); describeSkipIfNotMultiple('with bucketMatch set to false', () => { beforeEach(done => { - const command = new PutObjectCommand({ - Bucket: bucket, - Key: mismatchObject, + const command = new PutObjectCommand({ + Bucket: bucket, + Key: mismatchObject, Body: body, - Metadata: { 'scal-location-constraint': awsLocationMismatch } + Metadata: { 'scal-location-constraint': awsLocationMismatch }, }); s3.send(command) .then(() => done()) diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/get/getAwsVersioning.js b/tests/functional/aws-node-sdk/test/multipleBackend/get/getAwsVersioning.js index 5f514e2bca..6424c809ec 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/get/getAwsVersioning.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/get/getAwsVersioning.js @@ -1,10 +1,7 @@ const assert = require('assert'); const async = require('async'); const withV4 = require('../../support/withV4'); -const { GetObjectCommand, - PutObjectCommand, - CreateBucketCommand, - DeleteObjectCommand } = require('@aws-sdk/client-s3'); +const { GetObjectCommand, PutObjectCommand, CreateBucketCommand, DeleteObjectCommand } = require('@aws-sdk/client-s3'); const BucketUtility = require('../../../lib/utility/bucket-util'); const { awsS3, @@ -23,34 +20,36 @@ const { const someBody = 'testbody'; const bucket = `getawsversioning${genUniqID()}`; -function getAndAssertVersions(s3, bucket, key, versionIds, expectedData, - cb) { - async.mapSeries(versionIds, (versionId, next) => { - s3.send(new GetObjectCommand({ Bucket: bucket, Key: key, - VersionId: versionId })).then(async result => { - const resultBody = await result.Body.transformToString(); - next(null, { - VersionId: result.VersionId, - Body: resultBody +function getAndAssertVersions(s3, bucket, key, versionIds, expectedData, cb) { + async.mapSeries( + versionIds, + (versionId, next) => { + s3.send(new GetObjectCommand({ Bucket: bucket, Key: key, VersionId: versionId })) + .then(async result => { + const resultBody = await result.Body.transformToString(); + next(null, { + VersionId: result.VersionId, + Body: resultBody, + }); + }) + .catch(err => { + next(err); }); - }) - .catch(err => { - next(err); - }); - }, (err, results) => { - if (err) { - return cb(err); - } - const resultIds = results.map(result => result.VersionId); - const resultData = results.map(result => result.Body); - assert.deepStrictEqual(resultIds, versionIds); - assert.deepStrictEqual(resultData, expectedData); - return cb(); - }); + }, + (err, results) => { + if (err) { + return cb(err); + } + const resultIds = results.map(result => result.VersionId); + const resultData = results.map(result => result.Body); + assert.deepStrictEqual(resultIds, versionIds); + assert.deepStrictEqual(resultData, expectedData); + return cb(); + }, + ); } -describeSkipIfNotMultiple('AWS backend get object with versioning', -function testSuite() { +describeSkipIfNotMultiple('AWS backend get object with versioning', function testSuite() { this.timeout(30000); withV4(sigCfg => { let bucketUtil; @@ -60,8 +59,7 @@ function testSuite() { process.stdout.write('Creating bucket'); bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; - return s3.send(new CreateBucketCommand({ Bucket: bucket })) - .catch(err => { + return s3.send(new CreateBucketCommand({ Bucket: bucket })).catch(err => { process.stdout.write(`Error creating bucket: ${err}\n`); throw err; }); @@ -69,351 +67,582 @@ function testSuite() { afterEach(() => { process.stdout.write('Emptying bucket\n'); - return bucketUtil.empty(bucket) - .then(() => { - process.stdout.write('Deleting bucket\n'); - return bucketUtil.deleteOne(bucket); - }) - .catch(err => { - process.stdout.write('Error emptying/deleting bucket: ' + - `${err}\n`); - throw err; - }); + return bucketUtil + .empty(bucket) + .then(() => { + process.stdout.write('Deleting bucket\n'); + return bucketUtil.deleteOne(bucket); + }) + .catch(err => { + process.stdout.write('Error emptying/deleting bucket: ' + `${err}\n`); + throw err; + }); }); - it('should not return version ids when versioning has not been ' + - 'configured via CloudServer', done => { + it('should not return version ids when versioning has not been ' + 'configured via CloudServer', done => { const key = `somekey-${genUniqID()}`; - s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: someBody, - Metadata: { 'scal-location-constraint': awsLocation } })).then(data => { - assert.strictEqual(data.VersionId, undefined); - getAndAssertResult(s3, { bucket, key, body: someBody, - expectedVersionId: false }, done); - }).catch(err => { - assert.strictEqual(err, null, 'Expected success ' + - `putting object, got error ${err}`); - done(); - }); + s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: someBody, + Metadata: { 'scal-location-constraint': awsLocation }, + }), + ) + .then(data => { + assert.strictEqual(data.VersionId, undefined); + getAndAssertResult(s3, { bucket, key, body: someBody, expectedVersionId: false }, done); + }) + .catch(err => { + assert.strictEqual(err, null, 'Expected success ' + `putting object, got error ${err}`); + done(); + }); }); - it('should not return version ids when versioning has not been ' + - 'configured via CloudServer, even when version id specified', done => { - const key = `somekey-${genUniqID()}`; - s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: someBody, - Metadata: { 'scal-location-constraint': awsLocation } })).then(data => { - assert.strictEqual(data.VersionId, undefined); - getAndAssertResult(s3, { bucket, key, body: someBody, - versionId: 'null', expectedVersionId: false }, done); - }).catch(err => { - assert.strictEqual(err, null, 'Expected success ' + - `putting object, got error ${err}`); - done(); - }); - }); + it( + 'should not return version ids when versioning has not been ' + + 'configured via CloudServer, even when version id specified', + done => { + const key = `somekey-${genUniqID()}`; + s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: someBody, + Metadata: { 'scal-location-constraint': awsLocation }, + }), + ) + .then(data => { + assert.strictEqual(data.VersionId, undefined); + getAndAssertResult( + s3, + { bucket, key, body: someBody, versionId: 'null', expectedVersionId: false }, + done, + ); + }) + .catch(err => { + assert.strictEqual(err, null, 'Expected success ' + `putting object, got error ${err}`); + done(); + }); + }, + ); - it('should return version id for null version when versioning ' + - 'has been configured via CloudServer', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: someBody, - Metadata: { 'scal-location-constraint': awsLocation } })).then(() => next()) - .catch(err => { - assert.strictEqual(err, null, 'Expected success ' + - `putting object, got error ${err}`); - next(err); - }), - next => enableVersioning(s3, bucket, next), - // get with version id specified - next => getAndAssertResult(s3, { bucket, key, body: someBody, - versionId: 'null', expectedVersionId: 'null' }, next), - // get without version id specified - next => getAndAssertResult(s3, { bucket, key, body: someBody, - expectedVersionId: 'null' }, next), - ], done); - }); + it( + 'should return version id for null version when versioning ' + 'has been configured via CloudServer', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: someBody, + Metadata: { 'scal-location-constraint': awsLocation }, + }), + ) + .then(() => next()) + .catch(err => { + assert.strictEqual( + err, + null, + 'Expected success ' + `putting object, got error ${err}`, + ); + next(err); + }), + next => enableVersioning(s3, bucket, next), + // get with version id specified + next => + getAndAssertResult( + s3, + { bucket, key, body: someBody, versionId: 'null', expectedVersionId: 'null' }, + next, + ), + // get without version id specified + next => + getAndAssertResult(s3, { bucket, key, body: someBody, expectedVersionId: 'null' }, next), + ], + done, + ); + }, + ); - it('should overwrite the null version if putting object twice ' + - 'before versioning is configured', done => { + it('should overwrite the null version if putting object twice ' + 'before versioning is configured', done => { const key = `somekey-${genUniqID()}`; const data = ['data1', 'data2']; - async.waterfall([ - next => mapToAwsPuts(s3, bucket, key, data, err => next(err)), - // get latest version - next => getAndAssertResult(s3, { bucket, key, body: data[1], - expectedVersionId: false }, next), - // get specific version - next => getAndAssertResult(s3, { bucket, key, body: data[1], - versionId: 'null', expectedVersionId: false }, next), - ], done); + async.waterfall( + [ + next => mapToAwsPuts(s3, bucket, key, data, err => next(err)), + // get latest version + next => getAndAssertResult(s3, { bucket, key, body: data[1], expectedVersionId: false }, next), + // get specific version + next => + getAndAssertResult( + s3, + { bucket, key, body: data[1], versionId: 'null', expectedVersionId: false }, + next, + ), + ], + done, + ); }); - it('should overwrite existing null version if putting object ' + - 'after suspending versioning', done => { + it('should overwrite existing null version if putting object ' + 'after suspending versioning', done => { const key = `somekey-${genUniqID()}`; const data = ['data1', 'data2']; - async.waterfall([ - next => s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: data[0], - Metadata: { 'scal-location-constraint': awsLocation } })).then(() => next()) - .catch(err => { - assert.strictEqual(err, null, 'Expected success ' + - `putting object, got error ${err}`); - next(err); - }), - next => suspendVersioning(s3, bucket, next), - next => s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: data[1], - Metadata: { 'scal-location-constraint': awsLocation } })).then(() => next()) - .catch(err => { - assert.strictEqual(err, null, 'Expected success ' + - `putting object, got error ${err}`); - next(err); - }), - // get latest version - next => getAndAssertResult(s3, { bucket, key, body: data[1], - expectedVersionId: 'null' }, next), - // get specific version - next => getAndAssertResult(s3, { bucket, key, body: data[1], - versionId: 'null', expectedVersionId: 'null' }, next), - ], done); + async.waterfall( + [ + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: data[0], + Metadata: { 'scal-location-constraint': awsLocation }, + }), + ) + .then(() => next()) + .catch(err => { + assert.strictEqual(err, null, 'Expected success ' + `putting object, got error ${err}`); + next(err); + }), + next => suspendVersioning(s3, bucket, next), + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: data[1], + Metadata: { 'scal-location-constraint': awsLocation }, + }), + ) + .then(() => next()) + .catch(err => { + assert.strictEqual(err, null, 'Expected success ' + `putting object, got error ${err}`); + next(err); + }), + // get latest version + next => getAndAssertResult(s3, { bucket, key, body: data[1], expectedVersionId: 'null' }, next), + // get specific version + next => + getAndAssertResult( + s3, + { bucket, key, body: data[1], versionId: 'null', expectedVersionId: 'null' }, + next, + ), + ], + done, + ); }); - it('should overwrite null version if putting object when ' + - 'versioning is suspended after versioning enabled', done => { - const key = `somekey-${genUniqID()}`; - const data = [...Array(3).keys()].map(i => `data${i}`); - let firstVersionId; - async.waterfall([ - next => s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: data[0], - Metadata: { 'scal-location-constraint': awsLocation } })).then(() => next()) - .catch(err => { - assert.strictEqual(err, null, 'Expected success ' + - `putting object, got error ${err}`); - next(err); - }), - next => enableVersioning(s3, bucket, next), - next => s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: data[1], - Metadata: { 'scal-location-constraint': awsLocation } })).then(result => { - assert.notEqual(result.VersionId, 'null'); - firstVersionId = result.VersionId; - next(); - }).catch(err => { - assert.strictEqual(err, null, 'Expected success ' + - `putting object, got error ${err}`); - next(err); - }), - next => suspendVersioning(s3, bucket, next), - next => s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: data[3], - Metadata: { 'scal-location-constraint': awsLocation } })).then(() => next()) - .catch(err => { - assert.strictEqual(err, null, 'Expected success ' + - `putting object, got error ${err}`); - next(err); - }), - // get latest version - next => getAndAssertResult(s3, { bucket, key, body: data[3], - expectedVersionId: 'null' }, next), - // get specific version (null) - next => getAndAssertResult(s3, { bucket, key, body: data[3], - versionId: 'null', expectedVersionId: 'null' }, next), - // assert getting first version put for good measure - next => getAndAssertResult(s3, { bucket, key, body: data[1], - versionId: firstVersionId, - expectedVersionId: firstVersionId }, next), - ], done); - }); + it( + 'should overwrite null version if putting object when ' + + 'versioning is suspended after versioning enabled', + done => { + const key = `somekey-${genUniqID()}`; + const data = [...Array(3).keys()].map(i => `data${i}`); + let firstVersionId; + async.waterfall( + [ + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: data[0], + Metadata: { 'scal-location-constraint': awsLocation }, + }), + ) + .then(() => next()) + .catch(err => { + assert.strictEqual( + err, + null, + 'Expected success ' + `putting object, got error ${err}`, + ); + next(err); + }), + next => enableVersioning(s3, bucket, next), + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: data[1], + Metadata: { 'scal-location-constraint': awsLocation }, + }), + ) + .then(result => { + assert.notEqual(result.VersionId, 'null'); + firstVersionId = result.VersionId; + next(); + }) + .catch(err => { + assert.strictEqual( + err, + null, + 'Expected success ' + `putting object, got error ${err}`, + ); + next(err); + }), + next => suspendVersioning(s3, bucket, next), + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: data[3], + Metadata: { 'scal-location-constraint': awsLocation }, + }), + ) + .then(() => next()) + .catch(err => { + assert.strictEqual( + err, + null, + 'Expected success ' + `putting object, got error ${err}`, + ); + next(err); + }), + // get latest version + next => getAndAssertResult(s3, { bucket, key, body: data[3], expectedVersionId: 'null' }, next), + // get specific version (null) + next => + getAndAssertResult( + s3, + { bucket, key, body: data[3], versionId: 'null', expectedVersionId: 'null' }, + next, + ), + // assert getting first version put for good measure + next => + getAndAssertResult( + s3, + { + bucket, + key, + body: data[1], + versionId: firstVersionId, + expectedVersionId: firstVersionId, + }, + next, + ), + ], + done, + ); + }, + ); - it('should get correct data from aws backend using version IDs', - done => { + it('should get correct data from aws backend using version IDs', done => { const key = `somekey-${genUniqID()}`; const data = [...Array(5).keys()].map(i => i.toString()); const versionIds = ['null']; - async.waterfall([ - next => s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: data[0], - Metadata: { 'scal-location-constraint': awsLocation } })).then(() => next()) - .catch(err => { - next(err); - }), - next => putVersionsToAws(s3, bucket, key, data.slice(1), next), - (ids, next) => { - versionIds.push(...ids); - next(); - }, - next => getAndAssertVersions(s3, bucket, key, versionIds, data, - next), - ], done); + async.waterfall( + [ + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: data[0], + Metadata: { 'scal-location-constraint': awsLocation }, + }), + ) + .then(() => next()) + .catch(err => { + next(err); + }), + next => putVersionsToAws(s3, bucket, key, data.slice(1), next), + (ids, next) => { + versionIds.push(...ids); + next(); + }, + next => getAndAssertVersions(s3, bucket, key, versionIds, data, next), + ], + done, + ); }); - it('should get correct version when getting without version ID', - done => { + it('should get correct version when getting without version ID', done => { const key = `somekey-${genUniqID()}`; const data = [...Array(5).keys()].map(i => i.toString()); const versionIds = ['null']; - async.waterfall([ - next => s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: data[0], - Metadata: { 'scal-location-constraint': awsLocation } })).then(() => next()) - .catch(err => { - next(err); - }), - next => putVersionsToAws(s3, bucket, key, data.slice(1), next), - (ids, next) => { - versionIds.push(...ids); - next(); - }, - next => getAndAssertResult(s3, { bucket, key, body: data[4], - expectedVersionId: versionIds[4] }, next), - ], done); + async.waterfall( + [ + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: data[0], + Metadata: { 'scal-location-constraint': awsLocation }, + }), + ) + .then(() => next()) + .catch(err => { + next(err); + }), + next => putVersionsToAws(s3, bucket, key, data.slice(1), next), + (ids, next) => { + versionIds.push(...ids); + next(); + }, + next => + getAndAssertResult(s3, { bucket, key, body: data[4], expectedVersionId: versionIds[4] }, next), + ], + done, + ); }); - it('should get correct data from aws backend using version IDs ' + - 'after putting null versions, putting versions, putting more null ' + - 'versions and then putting more versions', - done => { - const key = `somekey-${genUniqID()}`; - const data = [...Array(16).keys()].map(i => i.toString()); - // put three null versions, - // 5 real versions, - // three null versions, - // 5 versions again - const firstThreeNullVersions = data.slice(0, 3); - const firstFiveVersions = data.slice(3, 8); - const secondThreeNullVersions = data.slice(8, 11); - const secondFiveVersions = data.slice(11, 16); - const versionIds = []; - const lastNullVersion = secondThreeNullVersions[2]; - const finalDataArr = firstFiveVersions.concat([lastNullVersion]) - .concat(secondFiveVersions); - async.waterfall([ - next => mapToAwsPuts(s3, bucket, key, firstThreeNullVersions, - err => next(err)), - next => putVersionsToAws(s3, bucket, key, firstFiveVersions, - next), - (ids, next) => { - versionIds.push(...ids); - next(); - }, - next => putNullVersionsToAws(s3, bucket, key, - secondThreeNullVersions, err => next(err)), - next => putVersionsToAws(s3, bucket, key, secondFiveVersions, - next), - (ids, next) => { - versionIds.push('null'); - versionIds.push(...ids); - next(); - }, - // get versions by id - next => getAndAssertVersions(s3, bucket, key, versionIds, - finalDataArr, next), - // get and assert latest version - next => getAndAssertResult(s3, { bucket, key, body: data[16], - versionId: versionIds[versionIds.length - 1], - expectedVersionId: versionIds[versionIds.length - 1] }, - next), - ], done); - }); + it( + 'should get correct data from aws backend using version IDs ' + + 'after putting null versions, putting versions, putting more null ' + + 'versions and then putting more versions', + done => { + const key = `somekey-${genUniqID()}`; + const data = [...Array(16).keys()].map(i => i.toString()); + // put three null versions, + // 5 real versions, + // three null versions, + // 5 versions again + const firstThreeNullVersions = data.slice(0, 3); + const firstFiveVersions = data.slice(3, 8); + const secondThreeNullVersions = data.slice(8, 11); + const secondFiveVersions = data.slice(11, 16); + const versionIds = []; + const lastNullVersion = secondThreeNullVersions[2]; + const finalDataArr = firstFiveVersions.concat([lastNullVersion]).concat(secondFiveVersions); + async.waterfall( + [ + next => mapToAwsPuts(s3, bucket, key, firstThreeNullVersions, err => next(err)), + next => putVersionsToAws(s3, bucket, key, firstFiveVersions, next), + (ids, next) => { + versionIds.push(...ids); + next(); + }, + next => putNullVersionsToAws(s3, bucket, key, secondThreeNullVersions, err => next(err)), + next => putVersionsToAws(s3, bucket, key, secondFiveVersions, next), + (ids, next) => { + versionIds.push('null'); + versionIds.push(...ids); + next(); + }, + // get versions by id + next => getAndAssertVersions(s3, bucket, key, versionIds, finalDataArr, next), + // get and assert latest version + next => + getAndAssertResult( + s3, + { + bucket, + key, + body: data[16], + versionId: versionIds[versionIds.length - 1], + expectedVersionId: versionIds[versionIds.length - 1], + }, + next, + ), + ], + done, + ); + }, + ); - it('should return the correct data getting versioned object ' + - 'even if object was deleted from AWS (creating a delete marker)', - done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => enableVersioning(s3, bucket, next), - next => s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: someBody, - Metadata: { 'scal-location-constraint': awsLocation } })).then(res => next(null, res.VersionId)) - .catch(err => { - next(err); - }), - // create a delete marker in AWS - (versionId, next) => awsS3.deleteObject({ Bucket: awsBucket, - Key: key }, err => next(err, versionId)), - (versionId, next) => getAndAssertResult(s3, { bucket, key, - body: someBody, expectedVersionId: versionId }, next), - ], done); - }); + it( + 'should return the correct data getting versioned object ' + + 'even if object was deleted from AWS (creating a delete marker)', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => enableVersioning(s3, bucket, next), + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: someBody, + Metadata: { 'scal-location-constraint': awsLocation }, + }), + ) + .then(res => next(null, res.VersionId)) + .catch(err => { + next(err); + }), + // create a delete marker in AWS + (versionId, next) => + awsS3.deleteObject({ Bucket: awsBucket, Key: key }, err => next(err, versionId)), + (versionId, next) => + getAndAssertResult(s3, { bucket, key, body: someBody, expectedVersionId: versionId }, next), + ], + done, + ); + }, + ); - it('should return the correct data getting versioned object ' + - 'even if object is put directly to AWS (creating new version)', - done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => enableVersioning(s3, bucket, next), - next => s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: someBody, - Metadata: { 'scal-location-constraint': awsLocation } })).then(res => next(null, res.VersionId)) - .catch(err => { - next(err); - }), - // put an object in AWS - (versionId, next) => awsS3.send(new PutObjectCommand({ Bucket: awsBucket, - Key: key })).then(() => next(null, versionId)) - .catch(err => { - next(err); - }), - (versionId, next) => getAndAssertResult(s3, { bucket, key, - body: someBody, expectedVersionId: versionId }, next), - ], done); - }); + it( + 'should return the correct data getting versioned object ' + + 'even if object is put directly to AWS (creating new version)', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => enableVersioning(s3, bucket, next), + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: someBody, + Metadata: { 'scal-location-constraint': awsLocation }, + }), + ) + .then(res => next(null, res.VersionId)) + .catch(err => { + next(err); + }), + // put an object in AWS + (versionId, next) => + awsS3 + .send(new PutObjectCommand({ Bucket: awsBucket, Key: key })) + .then(() => next(null, versionId)) + .catch(err => { + next(err); + }), + (versionId, next) => + getAndAssertResult(s3, { bucket, key, body: someBody, expectedVersionId: versionId }, next), + ], + done, + ); + }, + ); - it('should return a LocationNotFound if trying to get an object ' + - 'that was deleted in AWS but exists in s3 metadata', - done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => enableVersioning(s3, bucket, next), - next => s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: someBody, - Metadata: { 'scal-location-constraint': awsLocation } })).then(res => next(null, res.VersionId)) - .catch(err => { - next(err); - }), - // get the latest version id in aws - (s3vid, next) => awsS3.send(new GetObjectCommand({ Bucket: awsBucket, - Key: key })).then(res => next(null, s3vid, res.VersionId)) - .catch(err => { - next(err); - }), - (s3VerId, awsVerId, next) => awsS3.send(new DeleteObjectCommand({ - Bucket: awsBucket, Key: key, VersionId: awsVerId })).then(() => next(null, s3VerId)) - .catch(err => { - next(err); - }), - (s3VerId, next) => s3.send(new GetObjectCommand({ Bucket: bucket, Key: key })) - .then(res => next(null, s3VerId, res.VersionId)) - .catch(err => { - assert.strictEqual(err.name, 'LocationNotFound'); - assert.strictEqual(err.$metadata.httpStatusCode, 424); - next(); - }), - ], done); - }); + it( + 'should return a LocationNotFound if trying to get an object ' + + 'that was deleted in AWS but exists in s3 metadata', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => enableVersioning(s3, bucket, next), + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: someBody, + Metadata: { 'scal-location-constraint': awsLocation }, + }), + ) + .then(res => next(null, res.VersionId)) + .catch(err => { + next(err); + }), + // get the latest version id in aws + (s3vid, next) => + awsS3 + .send(new GetObjectCommand({ Bucket: awsBucket, Key: key })) + .then(res => next(null, s3vid, res.VersionId)) + .catch(err => { + next(err); + }), + (s3VerId, awsVerId, next) => + awsS3 + .send( + new DeleteObjectCommand({ + Bucket: awsBucket, + Key: key, + VersionId: awsVerId, + }), + ) + .then(() => next(null, s3VerId)) + .catch(err => { + next(err); + }), + (s3VerId, next) => + s3 + .send(new GetObjectCommand({ Bucket: bucket, Key: key })) + .then(res => next(null, s3VerId, res.VersionId)) + .catch(err => { + assert.strictEqual(err.name, 'LocationNotFound'); + assert.strictEqual(err.$metadata.httpStatusCode, 424); + next(); + }), + ], + done, + ); + }, + ); - it('should return a LocationNotFound if trying to get a version ' + - 'that was deleted in AWS but exists in s3 metadata', - done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => enableVersioning(s3, bucket, next), - next => s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: someBody, - Metadata: { 'scal-location-constraint': awsLocation } })).then(res => next(null, res.VersionId)) - .catch(err => { - next(err); - }), - // get the latest version id in aws - (s3vid, next) => awsS3.send(new GetObjectCommand({ Bucket: awsBucket, - Key: key })).then(res => next(null, s3vid, res.VersionId)) - .catch(err => { - next(err); - }), - (s3VerId, awsVerId, next) => awsS3.send(new DeleteObjectCommand({ - Bucket: awsBucket, Key: key, VersionId: awsVerId })).then(() => next(null, s3VerId)) - .catch(err => { - next(err); - }), - (s3VerId, next) => s3.send(new GetObjectCommand({ Bucket: bucket, Key: key, - VersionId: s3VerId })).then(() => { - next(); - }).catch(err => { - assert.strictEqual(err.name, 'LocationNotFound'); - assert.strictEqual(err.$metadata.httpStatusCode, 424); - next(); - }), - ], done); - }); + it( + 'should return a LocationNotFound if trying to get a version ' + + 'that was deleted in AWS but exists in s3 metadata', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => enableVersioning(s3, bucket, next), + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: someBody, + Metadata: { 'scal-location-constraint': awsLocation }, + }), + ) + .then(res => next(null, res.VersionId)) + .catch(err => { + next(err); + }), + // get the latest version id in aws + (s3vid, next) => + awsS3 + .send(new GetObjectCommand({ Bucket: awsBucket, Key: key })) + .then(res => next(null, s3vid, res.VersionId)) + .catch(err => { + next(err); + }), + (s3VerId, awsVerId, next) => + awsS3 + .send( + new DeleteObjectCommand({ + Bucket: awsBucket, + Key: key, + VersionId: awsVerId, + }), + ) + .then(() => next(null, s3VerId)) + .catch(err => { + next(err); + }), + (s3VerId, next) => + s3 + .send(new GetObjectCommand({ Bucket: bucket, Key: key, VersionId: s3VerId })) + .then(() => { + next(); + }) + .catch(err => { + assert.strictEqual(err.name, 'LocationNotFound'); + assert.strictEqual(err.$metadata.httpStatusCode, 424); + next(); + }), + ], + done, + ); + }, + ); }); }); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/get/getAzure.js b/tests/functional/aws-node-sdk/test/multipleBackend/get/getAzure.js index 7f1d5e25dc..a333b47ce0 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/get/getAzure.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/get/getAzure.js @@ -1,7 +1,5 @@ const assert = require('assert'); -const { CreateBucketCommand, - PutObjectCommand, - GetObjectCommand } = require('@aws-sdk/client-s3'); +const { CreateBucketCommand, PutObjectCommand, GetObjectCommand } = require('@aws-sdk/client-s3'); const BucketUtility = require('../../../lib/utility/bucket-util'); const withV4 = require('../../support/withV4'); @@ -24,8 +22,7 @@ const normalBody = Buffer.from('I am a body', 'utf8'); const azureTimeout = 10000; -describeSkipIfNotMultiple('Multiple backend get object from Azure', -function testSuite() { +describeSkipIfNotMultiple('Multiple backend get object from Azure', function testSuite() { this.timeout(30000); withV4(sigCfg => { let bucketUtil; @@ -35,8 +32,7 @@ function testSuite() { process.stdout.write('Creating bucket'); bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; - return s3.send(new CreateBucketCommand({ Bucket: azureContainerName })) - .catch(err => { + return s3.send(new CreateBucketCommand({ Bucket: azureContainerName })).catch(err => { process.stdout.write(`Error creating bucket: ${err}\n`); throw err; }); @@ -44,46 +40,49 @@ function testSuite() { after(() => { process.stdout.write('Emptying bucket\n'); - return bucketUtil.empty(azureContainerName) - .then(() => { - process.stdout.write('Deleting bucket\n'); - return bucketUtil.deleteOne(azureContainerName); - }) - .catch(err => { - process.stdout.write('Error emptying/deleting bucket: ' + - `${err}\n`); - throw err; - }); + return bucketUtil + .empty(azureContainerName) + .then(() => { + process.stdout.write('Deleting bucket\n'); + return bucketUtil.deleteOne(azureContainerName); + }) + .catch(err => { + process.stdout.write('Error emptying/deleting bucket: ' + `${err}\n`); + throw err; + }); }); keys.forEach(key => { describe(`${key.describe} size`, () => { const testKey = `${key.name}-${Date.now()}`; before(done => { setTimeout(() => { - s3.send(new PutObjectCommand({ - Bucket: azureContainerName, - Key: testKey, - Body: key.body, - Metadata: { - 'scal-location-constraint': azureLocation, - }, - })).then(() => done()) - .catch(err => { - done(err); - }); + s3.send( + new PutObjectCommand({ + Bucket: azureContainerName, + Key: testKey, + Body: key.body, + Metadata: { + 'scal-location-constraint': azureLocation, + }, + }), + ) + .then(() => done()) + .catch(err => { + done(err); + }); }, azureTimeout); }); it(`should get an ${key.describe} object from Azure`, done => { - s3.send(new GetObjectCommand({ Bucket: azureContainerName, Key: - testKey })).then(res => { - assert.strictEqual(res.ETag, `"${key.MD5}"`); - done(); - }).catch(err => { - assert.equal(err, null, 'Expected success ' + - `but got error ${err}`); - done(err); - }); + s3.send(new GetObjectCommand({ Bucket: azureContainerName, Key: testKey })) + .then(res => { + assert.strictEqual(res.ETag, `"${key.MD5}"`); + done(); + }) + .catch(err => { + assert.equal(err, null, 'Expected success ' + `but got error ${err}`); + done(err); + }); }); }); }); @@ -91,93 +90,106 @@ function testSuite() { describe('with range', () => { const azureObject = uniqName(keyObject); before(done => { - s3.send(new PutObjectCommand({ - Bucket: azureContainerName, - Key: azureObject, - Body: '0123456789', - Metadata: { - 'scal-location-constraint': azureLocation, - }, - })).then(() => done()) - .catch(err => { - done(err); - }); + s3.send( + new PutObjectCommand({ + Bucket: azureContainerName, + Key: azureObject, + Body: '0123456789', + Metadata: { + 'scal-location-constraint': azureLocation, + }, + }), + ) + .then(() => done()) + .catch(err => { + done(err); + }); }); - it('should get an object with body 012345 with "bytes=0-5"', - done => { - s3.send(new GetObjectCommand({ - Bucket: azureContainerName, - Key: azureObject, - Range: 'bytes=0-5', - })).then(async res => { - const body = await res.Body.transformToString(); - assert.equal(res.ContentLength, 6); - assert.strictEqual(res.ContentRange, 'bytes 0-5/10'); - assert.strictEqual(body, '012345'); - done(); - }).catch(err => { - assert.equal(err, null, 'Expected success but got ' + - `error ${err}`); - done(err); - }); + it('should get an object with body 012345 with "bytes=0-5"', done => { + s3.send( + new GetObjectCommand({ + Bucket: azureContainerName, + Key: azureObject, + Range: 'bytes=0-5', + }), + ) + .then(async res => { + const body = await res.Body.transformToString(); + assert.equal(res.ContentLength, 6); + assert.strictEqual(res.ContentRange, 'bytes 0-5/10'); + assert.strictEqual(body, '012345'); + done(); + }) + .catch(err => { + assert.equal(err, null, 'Expected success but got ' + `error ${err}`); + done(err); + }); }); - it('should get an object with body 456789 with "bytes=4-"', - done => { - s3.send(new GetObjectCommand({ - Bucket: azureContainerName, - Key: azureObject, - Range: 'bytes=4-', - })).then(async res => { - const body = await res.Body.transformToString(); - assert.equal(res.ContentLength, 6); - assert.strictEqual(res.ContentRange, 'bytes 4-9/10'); - assert.strictEqual(body, '456789'); - done(); - }).catch(err => { - assert.equal(err, null, 'Expected success but got ' + - `error ${err}`); - done(err); - }); + it('should get an object with body 456789 with "bytes=4-"', done => { + s3.send( + new GetObjectCommand({ + Bucket: azureContainerName, + Key: azureObject, + Range: 'bytes=4-', + }), + ) + .then(async res => { + const body = await res.Body.transformToString(); + assert.equal(res.ContentLength, 6); + assert.strictEqual(res.ContentRange, 'bytes 4-9/10'); + assert.strictEqual(body, '456789'); + done(); + }) + .catch(err => { + assert.equal(err, null, 'Expected success but got ' + `error ${err}`); + done(err); + }); }); }); describe('returning error', () => { const azureObject = uniqName(keyObject); before(done => { - s3.send(new PutObjectCommand({ - Bucket: azureContainerName, - Key: azureObject, - Body: normalBody, - Metadata: { - 'scal-location-constraint': azureLocation, - }, - })).then(() => { - azureClient.getContainerClient(azureContainerName) - .deleteBlob(azureObject).then(done, err => { - assert.equal(err, null, 'Expected success but got ' + - `error ${err}`); - done(err); - }); - }) - .catch(err => { - assert.equal(err, null, 'Expected success but got ' + - `error ${err}`); - done(err); - }); + s3.send( + new PutObjectCommand({ + Bucket: azureContainerName, + Key: azureObject, + Body: normalBody, + Metadata: { + 'scal-location-constraint': azureLocation, + }, + }), + ) + .then(() => { + azureClient + .getContainerClient(azureContainerName) + .deleteBlob(azureObject) + .then(done, err => { + assert.equal(err, null, 'Expected success but got ' + `error ${err}`); + done(err); + }); + }) + .catch(err => { + assert.equal(err, null, 'Expected success but got ' + `error ${err}`); + done(err); + }); }); - it('should return an error on get done to object deleted ' + - 'from Azure', done => { - s3.send(new GetObjectCommand({ - Bucket: azureContainerName, - Key: azureObject, - })).then(() => { - done(); - }).catch(err => { - assert.strictEqual(err.name, 'LocationNotFound'); - done(err); - }); + it('should return an error on get done to object deleted ' + 'from Azure', done => { + s3.send( + new GetObjectCommand({ + Bucket: azureContainerName, + Key: azureObject, + }), + ) + .then(() => { + done(); + }) + .catch(err => { + assert.strictEqual(err.name, 'LocationNotFound'); + done(err); + }); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/get/getGcp.js b/tests/functional/aws-node-sdk/test/multipleBackend/get/getGcp.js index a0895a7b48..acb5525a6b 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/get/getGcp.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/get/getGcp.js @@ -1,15 +1,8 @@ const assert = require('assert'); const withV4 = require('../../support/withV4'); -const { PutObjectCommand, - GetObjectCommand, - CreateBucketCommand } = require('@aws-sdk/client-s3'); +const { PutObjectCommand, GetObjectCommand, CreateBucketCommand } = require('@aws-sdk/client-s3'); const BucketUtility = require('../../../lib/utility/bucket-util'); -const { - gcpLocation, - gcpLocationMismatch, - genUniqID, - describeSkipIfNotMultiple, -} = require('../utils'); +const { gcpLocation, gcpLocationMismatch, genUniqID, describeSkipIfNotMultiple } = require('../utils'); const bucket = `getgcp${genUniqID()}`; const gcpObject = `gcpobject-${genUniqID()}`; @@ -32,8 +25,7 @@ describe('Multiple backend get object', function testSuite() { process.stdout.write('Creating bucket'); bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; - return s3.send(new CreateBucketCommand({ Bucket: bucket })) - .catch(err => { + return s3.send(new CreateBucketCommand({ Bucket: bucket })).catch(err => { process.stdout.write(`Error creating bucket: ${err}\n`); throw err; }); @@ -41,114 +33,128 @@ describe('Multiple backend get object', function testSuite() { after(() => { process.stdout.write('Emptying bucket\n'); - return bucketUtil.empty(bucket) - .then(() => { - process.stdout.write('Deleting bucket\n'); - return bucketUtil.deleteOne(bucket); - }) - .catch(err => { - process.stdout.write('Error emptying/deleting bucket: ' + - `${err}\n`); - throw err; - }); + return bucketUtil + .empty(bucket) + .then(() => { + process.stdout.write('Deleting bucket\n'); + return bucketUtil.deleteOne(bucket); + }) + .catch(err => { + process.stdout.write('Error emptying/deleting bucket: ' + `${err}\n`); + throw err; + }); }); describeSkipIfNotMultiple('with objects in GCP', () => { before(() => { process.stdout.write('Putting object to GCP\n'); - return s3.send(new PutObjectCommand({ Bucket: bucket, Key: gcpObject, - Body: body, - Metadata: { 'scal-location-constraint': gcpLocation }, - }) - .then(() => { - process.stdout.write('Putting 0-byte object to GCP\n'); - return s3.send(new PutObjectCommand({ Bucket: bucket, - Key: emptyGcpObject, + return s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: gcpObject, + Body: body, Metadata: { 'scal-location-constraint': gcpLocation }, - })); - }) - .then(() => { - process.stdout.write('Putting large object to GCP\n'); - return s3.send(new PutObjectCommand({ Bucket: bucket, - Key: bigObject, Body: bigBody, - Metadata: { 'scal-location-constraint': gcpLocation }, - })); - }) - .catch(err => { - process.stdout.write(`Error putting objects: ${err}\n`); - throw err; - })); + }) + .then(() => { + process.stdout.write('Putting 0-byte object to GCP\n'); + return s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: emptyGcpObject, + Metadata: { 'scal-location-constraint': gcpLocation }, + }), + ); + }) + .then(() => { + process.stdout.write('Putting large object to GCP\n'); + return s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: bigObject, + Body: bigBody, + Metadata: { 'scal-location-constraint': gcpLocation }, + }), + ); + }) + .catch(err => { + process.stdout.write(`Error putting objects: ${err}\n`); + throw err; + }), + ); }); const getTests = [ { msg: 'should get a 0-byte object from GCP', - input: { Bucket: bucket, Key: emptyGcpObject, - range: null, size: null }, + input: { Bucket: bucket, Key: emptyGcpObject, range: null, size: null }, output: { MD5: emptyMD5, contentRange: null }, }, { msg: 'should get an object from GCP', - input: { Bucket: bucket, Key: gcpObject, - range: null, size: null }, + input: { Bucket: bucket, Key: gcpObject, range: null, size: null }, output: { MD5: correctMD5, contentRange: null }, }, { msg: 'should get a large object from GCP', - input: { Bucket: bucket, Key: bigObject, - range: null, size: null }, + input: { Bucket: bucket, Key: bigObject, range: null, size: null }, output: { MD5: bigMD5, contentRange: null }, }, { msg: 'should get an object using range query from GCP', - input: { Bucket: bucket, Key: bigObject, - range: 'bytes=0-9', size: 10 }, - output: { MD5: bigMD5, - contentRange: `bytes 0-9/${bigBodyLen}` }, + input: { Bucket: bucket, Key: bigObject, range: 'bytes=0-9', size: 10 }, + output: { MD5: bigMD5, contentRange: `bytes 0-9/${bigBodyLen}` }, }, ]; getTests.forEach(test => { const { Bucket, Key, range, size } = test.input; const { MD5, contentRange } = test.output; it(test.msg, done => { - s3.send(new GetObjectCommand({ Bucket, Key, Range: range })).then(res => { - if (range) { - assert.strictEqual(res.ContentLength, size); - assert.strictEqual(res.ContentRange, contentRange); - } - assert.strictEqual(res.ETag, `"${MD5}"`); - done(); - }) - .catch(err => { - assert.equal(err, null, - `Expected success but got error ${err}`); - done(err); - }); + s3.send(new GetObjectCommand({ Bucket, Key, Range: range })) + .then(res => { + if (range) { + assert.strictEqual(res.ContentLength, size); + assert.strictEqual(res.ContentRange, contentRange); + } + assert.strictEqual(res.ETag, `"${MD5}"`); + done(); + }) + .catch(err => { + assert.equal(err, null, `Expected success but got error ${err}`); + done(err); + }); }); }); }); describeSkipIfNotMultiple('with bucketMatch set to false', () => { beforeEach(done => { - s3.send(new PutObjectCommand({ Bucket: bucket, Key: mismatchObject, Body: body, - Metadata: { 'scal-location-constraint': gcpLocationMismatch } })).then(() => { - done(); - }) - .catch(err => { - assert.equal(err, null, `Err putting object: ${err}`); - done(err); - }); + s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: mismatchObject, + Body: body, + Metadata: { 'scal-location-constraint': gcpLocationMismatch }, + }), + ) + .then(() => { + done(); + }) + .catch(err => { + assert.equal(err, null, `Err putting object: ${err}`); + done(err); + }); }); it('should get an object from GCP', done => { - s3.send(new GetObjectCommand({ Bucket: bucket, Key: mismatchObject })).then(res => { - assert.strictEqual(res.ETag, `"${correctMD5}"`); - done(); - }) - .catch(err => { - assert.equal(err, null, `Error getting object: ${err}`); - done(err); - }); + s3.send(new GetObjectCommand({ Bucket: bucket, Key: mismatchObject })) + .then(res => { + assert.strictEqual(res.ETag, `"${correctMD5}"`); + done(); + }) + .catch(err => { + assert.equal(err, null, `Error getting object: ${err}`); + done(err); + }); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/initMPU/initMPUAzure.js b/tests/functional/aws-node-sdk/test/multipleBackend/initMPU/initMPUAzure.js index 33c131ff31..4b99c9fc7b 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/initMPU/initMPUAzure.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/initMPU/initMPUAzure.js @@ -9,8 +9,7 @@ const { const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); -const { azureLocation, getAzureContainerName, - genUniqID, describeSkipIfNotMultiple } = require('../utils'); +const { azureLocation, getAzureContainerName, genUniqID, describeSkipIfNotMultiple } = require('../utils'); const keyName = `somekey-${genUniqID()}`; @@ -27,26 +26,31 @@ describeSkipIfNotMultiple('Initiate MPU to AZURE', () => { afterEach(() => { process.stdout.write('Emptying bucket\n'); - return bucketUtil.empty(azureContainerName) - .then(() => { - process.stdout.write('Deleting bucket\n'); - return bucketUtil.deleteOne(azureContainerName); - }) - .catch(err => { - process.stdout.write(`Error in afterEach: ${err}\n`); - throw err; - }); + return bucketUtil + .empty(azureContainerName) + .then(() => { + process.stdout.write('Deleting bucket\n'); + return bucketUtil.deleteOne(azureContainerName); + }) + .catch(err => { + process.stdout.write(`Error in afterEach: ${err}\n`); + throw err; + }); }); describe('Basic test: ', () => { beforeEach(done => - s3.send(new CreateBucketCommand({ - Bucket: azureContainerName, - CreateBucketConfiguration: { - LocationConstraint: azureLocation, - }, - })) - .then(() => done()) - .catch(done)); + s3 + .send( + new CreateBucketCommand({ + Bucket: azureContainerName, + CreateBucketConfiguration: { + LocationConstraint: azureLocation, + }, + }), + ) + .then(() => done()) + .catch(done), + ); afterEach(function afterEachF(done) { const params = { Bucket: azureContainerName, @@ -57,43 +61,43 @@ describeSkipIfNotMultiple('Initiate MPU to AZURE', () => { .then(() => done()) .catch(done); }); - it('should create MPU and list in-progress multipart uploads', - function ifF(done) { + it('should create MPU and list in-progress multipart uploads', function ifF(done) { const params = { Bucket: azureContainerName, Key: keyName, Metadata: { 'scal-location-constraint': azureLocation }, }; - async.waterfall([ - next => { - s3.send(new CreateMultipartUploadCommand(params)) - .then(res => { - this.test.uploadId = res.UploadId; - assert(this.test.uploadId); - assert.strictEqual(res.Bucket, - azureContainerName); - assert.strictEqual(res.Key, keyName); - next(); - }) - .catch(next); - }, - next => { - s3.send(new ListMultipartUploadsCommand({ - Bucket: azureContainerName, - })) - .then(res => { - assert.strictEqual(res.NextKeyMarker, keyName); - assert.strictEqual(res.NextUploadIdMarker, - this.test.uploadId); - assert.strictEqual(res.Uploads[0].Key, - keyName); - assert.strictEqual(res.Uploads[0].UploadId, - this.test.uploadId); - next(); - }) - .catch(next); - }, - ], done); + async.waterfall( + [ + next => { + s3.send(new CreateMultipartUploadCommand(params)) + .then(res => { + this.test.uploadId = res.UploadId; + assert(this.test.uploadId); + assert.strictEqual(res.Bucket, azureContainerName); + assert.strictEqual(res.Key, keyName); + next(); + }) + .catch(next); + }, + next => { + s3.send( + new ListMultipartUploadsCommand({ + Bucket: azureContainerName, + }), + ) + .then(res => { + assert.strictEqual(res.NextKeyMarker, keyName); + assert.strictEqual(res.NextUploadIdMarker, this.test.uploadId); + assert.strictEqual(res.Uploads[0].Key, keyName); + assert.strictEqual(res.Uploads[0].UploadId, this.test.uploadId); + next(); + }) + .catch(next); + }, + ], + done, + ); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/initMPU/initMPUGcp.js b/tests/functional/aws-node-sdk/test/multipleBackend/initMPU/initMPUGcp.js index e2075dbd12..c8b7277d74 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/initMPU/initMPUGcp.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/initMPU/initMPUGcp.js @@ -10,8 +10,7 @@ const { const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); -const { gcpClient, gcpBucketMPU, gcpLocation, - genUniqID, describeSkipIfNotMultiple } = require('../utils'); +const { gcpClient, gcpBucketMPU, gcpLocation, genUniqID, describeSkipIfNotMultiple } = require('../utils'); const { createMpuKey } = arsenal.storage.data.external.GcpUtils; const bucket = `initmpugcp${genUniqID()}`; @@ -29,26 +28,31 @@ describeSkipIfNotMultiple('Initiate MPU to GCP', () => { afterEach(() => { process.stdout.write('Emptying bucket\n'); - return bucketUtil.empty(bucket) - .then(() => { - process.stdout.write('Deleting bucket\n'); - return bucketUtil.deleteOne(bucket); - }) - .catch(err => { - process.stdout.write(`Error in afterEach: ${err}\n`); - throw err; - }); + return bucketUtil + .empty(bucket) + .then(() => { + process.stdout.write('Deleting bucket\n'); + return bucketUtil.deleteOne(bucket); + }) + .catch(err => { + process.stdout.write(`Error in afterEach: ${err}\n`); + throw err; + }); }); describe('Basic test: ', () => { beforeEach(done => - s3.send(new CreateBucketCommand({ - Bucket: bucket, - CreateBucketConfiguration: { - LocationConstraint: gcpLocation, - }, - })) - .then(() => done()) - .catch(done)); + s3 + .send( + new CreateBucketCommand({ + Bucket: bucket, + CreateBucketConfiguration: { + LocationConstraint: gcpLocation, + }, + }), + ) + .then(() => done()) + .catch(done), + ); afterEach(function afterEachF(done) { const params = { Bucket: bucket, @@ -59,54 +63,54 @@ describeSkipIfNotMultiple('Initiate MPU to GCP', () => { .then(() => done()) .catch(done); }); - it('should create MPU and list in-progress multipart uploads', - function ifF(done) { + it('should create MPU and list in-progress multipart uploads', function ifF(done) { const params = { Bucket: bucket, Key: keyName, Metadata: { 'scal-location-constraint': gcpLocation }, }; - async.waterfall([ - next => { - s3.send(new CreateMultipartUploadCommand(params)) - .then(res => { - this.test.uploadId = res.UploadId; - assert(this.test.uploadId); - assert.strictEqual(res.Bucket, bucket); - assert.strictEqual(res.Key, keyName); - next(); - }) - .catch(next); - }, - next => { - s3.send(new ListMultipartUploadsCommand({ - Bucket: bucket, - })) - .then(res => { - assert.strictEqual(res.NextKeyMarker, keyName); - assert.strictEqual(res.NextUploadIdMarker, - this.test.uploadId); - assert.strictEqual(res.Uploads[0].Key, keyName); - assert.strictEqual(res.Uploads[0].UploadId, - this.test.uploadId); + async.waterfall( + [ + next => { + s3.send(new CreateMultipartUploadCommand(params)) + .then(res => { + this.test.uploadId = res.UploadId; + assert(this.test.uploadId); + assert.strictEqual(res.Bucket, bucket); + assert.strictEqual(res.Key, keyName); + next(); + }) + .catch(next); + }, + next => { + s3.send( + new ListMultipartUploadsCommand({ + Bucket: bucket, + }), + ) + .then(res => { + assert.strictEqual(res.NextKeyMarker, keyName); + assert.strictEqual(res.NextUploadIdMarker, this.test.uploadId); + assert.strictEqual(res.Uploads[0].Key, keyName); + assert.strictEqual(res.Uploads[0].UploadId, this.test.uploadId); + next(); + }) + .catch(next); + }, + next => { + const mpuKey = createMpuKey(keyName, this.test.uploadId, 'init'); + const params = { + Bucket: gcpBucketMPU, + Key: mpuKey, + }; + gcpClient.getObject(params, err => { + assert.ifError(err, `Expected success, but got err ${err}`); next(); - }) - .catch(next); - }, - next => { - const mpuKey = - createMpuKey(keyName, this.test.uploadId, 'init'); - const params = { - Bucket: gcpBucketMPU, - Key: mpuKey, - }; - gcpClient.getObject(params, err => { - assert.ifError(err, - `Expected success, but got err ${err}`); - next(); - }); - }, - ], done); + }); + }, + ], + done, + ); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/listParts/azureListParts.js b/tests/functional/aws-node-sdk/test/multipleBackend/listParts/azureListParts.js index 385b60b523..1e129fe76c 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/listParts/azureListParts.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/listParts/azureListParts.js @@ -1,14 +1,15 @@ const assert = require('assert'); -const { CreateBucketCommand, +const { + CreateBucketCommand, CreateMultipartUploadCommand, UploadPartCommand, AbortMultipartUploadCommand, - ListPartsCommand } = require('@aws-sdk/client-s3'); + ListPartsCommand, +} = require('@aws-sdk/client-s3'); const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); -const { azureLocation, getAzureContainerName, - genUniqID, describeSkipIfNotMultiple } = require('../utils'); +const { azureLocation, getAzureContainerName, genUniqID, describeSkipIfNotMultiple } = require('../utils'); const azureContainerName = getAzureContainerName(azureLocation); const firstPartSize = 10; @@ -19,88 +20,121 @@ const bodySecondPart = Buffer.alloc(secondPartSize); let bucketUtil; let s3; -describeSkipIfNotMultiple('List parts of MPU on Azure data backend', -() => { +describeSkipIfNotMultiple('List parts of MPU on Azure data backend', () => { withV4(sigCfg => { beforeEach(function beforeEachFn() { this.currentTest.key = `somekey-${genUniqID()}`; bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; - return s3.send(new CreateBucketCommand({ Bucket: azureContainerName })) - .then(() => s3.send(new CreateMultipartUploadCommand({ - Bucket: azureContainerName, Key: this.currentTest.key, - Metadata: { 'scal-location-constraint': azureLocation }, - })) - .then(res => { - this.currentTest.uploadId = res.UploadId; - return s3.send(new UploadPartCommand({ Bucket: azureContainerName, - Key: this.currentTest.key, PartNumber: 1, - UploadId: this.currentTest.uploadId, Body: bodyFirstPart, - })); - }).then(res => { - this.currentTest.firstEtag = res.ETag; - }).then(() => s3.send(new UploadPartCommand({ Bucket: azureContainerName, - Key: this.currentTest.key, PartNumber: 2, - UploadId: this.currentTest.uploadId, Body: bodySecondPart, - }))).then(res => { - this.currentTest.secondEtag = res.ETag; - }) - .catch(err => { - process.stdout.write(`Error in beforeEach: ${err}\n`); - throw err; - })); + return s3.send(new CreateBucketCommand({ Bucket: azureContainerName })).then(() => + s3 + .send( + new CreateMultipartUploadCommand({ + Bucket: azureContainerName, + Key: this.currentTest.key, + Metadata: { 'scal-location-constraint': azureLocation }, + }), + ) + .then(res => { + this.currentTest.uploadId = res.UploadId; + return s3.send( + new UploadPartCommand({ + Bucket: azureContainerName, + Key: this.currentTest.key, + PartNumber: 1, + UploadId: this.currentTest.uploadId, + Body: bodyFirstPart, + }), + ); + }) + .then(res => { + this.currentTest.firstEtag = res.ETag; + }) + .then(() => + s3.send( + new UploadPartCommand({ + Bucket: azureContainerName, + Key: this.currentTest.key, + PartNumber: 2, + UploadId: this.currentTest.uploadId, + Body: bodySecondPart, + }), + ), + ) + .then(res => { + this.currentTest.secondEtag = res.ETag; + }) + .catch(err => { + process.stdout.write(`Error in beforeEach: ${err}\n`); + throw err; + }), + ); }); afterEach(function afterEachFn() { process.stdout.write('Emptying bucket'); - return s3.send(new AbortMultipartUploadCommand({ - Bucket: azureContainerName, Key: this.currentTest.key, - UploadId: this.currentTest.uploadId, - })) - .then(() => bucketUtil.empty(azureContainerName)) - .then(() => { - process.stdout.write('Deleting bucket'); - return bucketUtil.deleteOne(azureContainerName); - }) - .catch(err => { - process.stdout.write('Error in afterEach'); - throw err; - }); + return s3 + .send( + new AbortMultipartUploadCommand({ + Bucket: azureContainerName, + Key: this.currentTest.key, + UploadId: this.currentTest.uploadId, + }), + ) + .then(() => bucketUtil.empty(azureContainerName)) + .then(() => { + process.stdout.write('Deleting bucket'); + return bucketUtil.deleteOne(azureContainerName); + }) + .catch(err => { + process.stdout.write('Error in afterEach'); + throw err; + }); }); it('should list both parts', function itFn(done) { - s3.send(new ListPartsCommand({ - Bucket: azureContainerName, - Key: this.test.key, - UploadId: this.test.uploadId })).then(data => { - assert.strictEqual(data.Parts.length, 2); - assert.strictEqual(data.Parts[0].PartNumber, 1); - assert.strictEqual(data.Parts[0].Size, firstPartSize); - assert.strictEqual(data.Parts[0].ETag, this.test.firstEtag); - assert.strictEqual(data.Parts[1].PartNumber, 2); - assert.strictEqual(data.Parts[1].Size, secondPartSize); - assert.strictEqual(data.Parts[1].ETag, this.test.secondEtag); - done(); - }).catch(err => { - assert.equal(err, null, `Err listing parts: ${err}`); - done(err); - }); + s3.send( + new ListPartsCommand({ + Bucket: azureContainerName, + Key: this.test.key, + UploadId: this.test.uploadId, + }), + ) + .then(data => { + assert.strictEqual(data.Parts.length, 2); + assert.strictEqual(data.Parts[0].PartNumber, 1); + assert.strictEqual(data.Parts[0].Size, firstPartSize); + assert.strictEqual(data.Parts[0].ETag, this.test.firstEtag); + assert.strictEqual(data.Parts[1].PartNumber, 2); + assert.strictEqual(data.Parts[1].Size, secondPartSize); + assert.strictEqual(data.Parts[1].ETag, this.test.secondEtag); + done(); + }) + .catch(err => { + assert.equal(err, null, `Err listing parts: ${err}`); + done(err); + }); }); it('should only list the second part', function itFn(done) { - s3.send(new ListPartsCommand({ - Bucket: azureContainerName, - Key: this.test.key, - PartNumberMarker: 1, - UploadId: this.test.uploadId })).then(data => { - assert.strictEqual(data.Parts[0].PartNumber, 2); - assert.strictEqual(data.Parts[0].Size, secondPartSize); - assert.strictEqual(data.Parts[0].ETag, this.test.secondEtag); - done(); - }).catch(err => { - assert.equal(err, null, `Err listing parts: ${err}`); - done(err); - }); + s3.send( + new ListPartsCommand({ + Bucket: azureContainerName, + Key: this.test.key, + PartNumberMarker: 1, + UploadId: this.test.uploadId, + }), + ) + .then(data => { + assert.strictEqual(data.Parts[0].PartNumber, 2); + assert.strictEqual(data.Parts[0].Size, secondPartSize); + assert.strictEqual(data.Parts[0].ETag, this.test.secondEtag); + done(); + }) + .catch(err => { + assert.equal(err, null, `Err listing parts: ${err}`); + done(err); + }); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/listParts/listPartsGcp.js b/tests/functional/aws-node-sdk/test/multipleBackend/listParts/listPartsGcp.js index d1093d04f5..fecfa76383 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/listParts/listPartsGcp.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/listParts/listPartsGcp.js @@ -1,13 +1,14 @@ const assert = require('assert'); -const { CreateBucketCommand, +const { + CreateBucketCommand, CreateMultipartUploadCommand, UploadPartCommand, AbortMultipartUploadCommand, - ListPartsCommand } = require('@aws-sdk/client-s3'); + ListPartsCommand, +} = require('@aws-sdk/client-s3'); const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); -const { gcpLocation, genUniqID, describeSkipIfNotMultiple } - = require('../utils'); +const { gcpLocation, genUniqID, describeSkipIfNotMultiple } = require('../utils'); const bucket = `listpartsgcp${genUniqID()}`; const firstPartSize = 10; @@ -24,82 +25,116 @@ describeSkipIfNotMultiple('List parts of MPU on GCP data backend', () => { this.currentTest.key = `somekey-${genUniqID()}`; bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; - return s3.send(new CreateBucketCommand({ Bucket: bucket })) - .then(() => s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, Key: this.currentTest.key, - Metadata: { 'scal-location-constraint': gcpLocation }, - }))) - .then(res => { - this.currentTest.uploadId = res.UploadId; - return s3.send(new UploadPartCommand({ Bucket: bucket, - Key: this.currentTest.key, PartNumber: 1, - UploadId: this.currentTest.uploadId, Body: bodyFirstPart, - })); - }).then(res => { - this.currentTest.firstEtag = res.ETag; - }).then(() => s3.send(new UploadPartCommand({ Bucket: bucket, - Key: this.currentTest.key, PartNumber: 2, - UploadId: this.currentTest.uploadId, Body: bodySecondPart, - }))) - .then(res => { - this.currentTest.secondEtag = res.ETag; - }) - .catch(err => { - process.stdout.write(`Error in beforeEach: ${err}\n`); - throw err; - }); + return s3 + .send(new CreateBucketCommand({ Bucket: bucket })) + .then(() => + s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: this.currentTest.key, + Metadata: { 'scal-location-constraint': gcpLocation }, + }), + ), + ) + .then(res => { + this.currentTest.uploadId = res.UploadId; + return s3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: this.currentTest.key, + PartNumber: 1, + UploadId: this.currentTest.uploadId, + Body: bodyFirstPart, + }), + ); + }) + .then(res => { + this.currentTest.firstEtag = res.ETag; + }) + .then(() => + s3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: this.currentTest.key, + PartNumber: 2, + UploadId: this.currentTest.uploadId, + Body: bodySecondPart, + }), + ), + ) + .then(res => { + this.currentTest.secondEtag = res.ETag; + }) + .catch(err => { + process.stdout.write(`Error in beforeEach: ${err}\n`); + throw err; + }); }); afterEach(function afterEachFn() { process.stdout.write('Emptying bucket'); - return s3.send(new AbortMultipartUploadCommand({ - Bucket: bucket, Key: this.currentTest.key, - UploadId: this.currentTest.uploadId, - })) - .then(() => bucketUtil.empty(bucket)) - .then(() => { - process.stdout.write('Deleting bucket'); - return bucketUtil.deleteOne(bucket); - }) - .catch(err => { - process.stdout.write('Error in afterEach'); - throw err; - }); + return s3 + .send( + new AbortMultipartUploadCommand({ + Bucket: bucket, + Key: this.currentTest.key, + UploadId: this.currentTest.uploadId, + }), + ) + .then(() => bucketUtil.empty(bucket)) + .then(() => { + process.stdout.write('Deleting bucket'); + return bucketUtil.deleteOne(bucket); + }) + .catch(err => { + process.stdout.write('Error in afterEach'); + throw err; + }); }); it('should list both parts', function itFn(done) { - s3.send(new ListPartsCommand({ - Bucket: bucket, - Key: this.test.key, - UploadId: this.test.uploadId })).then(data => { - assert.strictEqual(data.Parts.length, 2); - assert.strictEqual(data.Parts[0].PartNumber, 1); - assert.strictEqual(data.Parts[0].Size, firstPartSize); - assert.strictEqual(data.Parts[0].ETag, this.test.firstEtag); - assert.strictEqual(data.Parts[1].PartNumber, 2); - assert.strictEqual(data.Parts[1].Size, secondPartSize); - assert.strictEqual(data.Parts[1].ETag, this.test.secondEtag); - done(); - }).catch(err => { - assert.equal(err, null, `Err listing parts: ${err}`); - done(err); - }); + s3.send( + new ListPartsCommand({ + Bucket: bucket, + Key: this.test.key, + UploadId: this.test.uploadId, + }), + ) + .then(data => { + assert.strictEqual(data.Parts.length, 2); + assert.strictEqual(data.Parts[0].PartNumber, 1); + assert.strictEqual(data.Parts[0].Size, firstPartSize); + assert.strictEqual(data.Parts[0].ETag, this.test.firstEtag); + assert.strictEqual(data.Parts[1].PartNumber, 2); + assert.strictEqual(data.Parts[1].Size, secondPartSize); + assert.strictEqual(data.Parts[1].ETag, this.test.secondEtag); + done(); + }) + .catch(err => { + assert.equal(err, null, `Err listing parts: ${err}`); + done(err); + }); }); it('should only list the second part', function itFn(done) { - s3.send(new ListPartsCommand({ - Bucket: bucket, - Key: this.test.key, - PartNumberMarker: 1, - UploadId: this.test.uploadId })).then(data => { - assert.strictEqual(data.Parts[0].PartNumber, 2); - assert.strictEqual(data.Parts[0].Size, secondPartSize); - assert.strictEqual(data.Parts[0].ETag, this.test.secondEtag); - done(); - }).catch(err => { - assert.equal(err, null, `Err listing parts: ${err}`); - done(err); - }); + s3.send( + new ListPartsCommand({ + Bucket: bucket, + Key: this.test.key, + PartNumberMarker: 1, + UploadId: this.test.uploadId, + }), + ) + .then(data => { + assert.strictEqual(data.Parts[0].PartNumber, 2); + assert.strictEqual(data.Parts[0].Size, secondPartSize); + assert.strictEqual(data.Parts[0].ETag, this.test.secondEtag); + done(); + }) + .catch(err => { + assert.equal(err, null, `Err listing parts: ${err}`); + done(err); + }); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/mpuAbort/abortMPUGcp.js b/tests/functional/aws-node-sdk/test/multipleBackend/mpuAbort/abortMPUGcp.js index 8133eafcfb..05b0f91ad1 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/mpuAbort/abortMPUGcp.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/mpuAbort/abortMPUGcp.js @@ -11,8 +11,15 @@ const { const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); -const { gcpClient, gcpBucket, gcpBucketMPU, - gcpLocation, uniqName, genUniqID, describeSkipIfNotMultiple } = require('../utils'); +const { + gcpClient, + gcpBucket, + gcpBucketMPU, + gcpLocation, + uniqName, + genUniqID, + describeSkipIfNotMultiple, +} = require('../utils'); const keyObject = 'abortgcp'; const bucket = `abortmpugcp${genUniqID()}`; @@ -30,16 +37,13 @@ function checkMPUList(bucket, key, uploadId, cb) { UploadId: uploadId, }; gcpClient.listParts(params, (err, res) => { - assert.ifError(err, - `Expected success, but got err ${err}`); - assert.deepStrictEqual(res.Contents, [], - 'Expected 0 parts, listed some'); + assert.ifError(err, `Expected success, but got err ${err}`); + assert.deepStrictEqual(res.Contents, [], 'Expected 0 parts, listed some'); cb(); }); } -describeSkipIfNotMultiple('Abort MPU on GCP data backend', function -descrbeFn() { +describeSkipIfNotMultiple('Abort MPU on GCP data backend', function descrbeFn() { this.timeout(180000); withV4(sigCfg => { beforeEach(function beforeFn() { @@ -50,33 +54,42 @@ descrbeFn() { describe('with bucket location header', () => { beforeEach(function beforeEachFn(done) { - async.waterfall([ - next => { - s3.send(new CreateBucketCommand({ - Bucket: bucket, - })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: this.currentTest.key, - Metadata: { 'scal-location-constraint': gcpLocation }, - })) - .then(res => { - this.currentTest.uploadId = res.UploadId; - next(); - }) - .catch(next); - }, - ], done); + async.waterfall( + [ + next => { + s3.send( + new CreateBucketCommand({ + Bucket: bucket, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: this.currentTest.key, + Metadata: { 'scal-location-constraint': gcpLocation }, + }), + ) + .then(res => { + this.currentTest.uploadId = res.UploadId; + next(); + }) + .catch(next); + }, + ], + done, + ); }); afterEach(done => { - s3.send(new DeleteBucketCommand({ - Bucket: bucket, - })) + s3.send( + new DeleteBucketCommand({ + Bucket: bucket, + }), + ) .then(() => done()) .catch(done); }); @@ -87,16 +100,21 @@ descrbeFn() { Key: this.test.key, UploadId: this.test.uploadId, }; - async.waterfall([ - next => { - s3.send(new AbortMultipartUploadCommand(params)) - .then(() => next()) - .catch(next); - }, - next => setTimeout(() => checkMPUList( - gcpBucketMPU, this.test.key, this.test.uploadId, next), - gcpTimeout), - ], done); + async.waterfall( + [ + next => { + s3.send(new AbortMultipartUploadCommand(params)) + .then(() => next()) + .catch(next); + }, + next => + setTimeout( + () => checkMPUList(gcpBucketMPU, this.test.key, this.test.uploadId, next), + gcpTimeout, + ), + ], + done, + ); }); it('should abort a MPU with uploaded parts', function itFn(done) { @@ -105,133 +123,149 @@ descrbeFn() { Key: this.test.key, UploadId: this.test.uploadId, }; - async.waterfall([ - next => { - async.times(2, (n, cb) => { - const uploadParams = { - Bucket: bucket, - Key: this.test.key, - UploadId: this.test.uploadId, - Body: body, - PartNumber: n + 1, - }; - s3.send(new UploadPartCommand(uploadParams)) - .then(res => { - assert.strictEqual( - res.ETag, `"${correctMD5}"`); - cb(); - }) - .catch(err => { - assert.ifError(err, - `Expected success, but got err ${err}`); - cb(err); - }); - }, err => next(err)); - }, - next => { - s3.send(new AbortMultipartUploadCommand(params)) - .then(() => next()) - .catch(next); - }, - next => setTimeout(() => checkMPUList( - gcpBucketMPU, this.test.key, this.test.uploadId, next), - gcpTimeout), - ], done); + async.waterfall( + [ + next => { + async.times( + 2, + (n, cb) => { + const uploadParams = { + Bucket: bucket, + Key: this.test.key, + UploadId: this.test.uploadId, + Body: body, + PartNumber: n + 1, + }; + s3.send(new UploadPartCommand(uploadParams)) + .then(res => { + assert.strictEqual(res.ETag, `"${correctMD5}"`); + cb(); + }) + .catch(err => { + assert.ifError(err, `Expected success, but got err ${err}`); + cb(err); + }); + }, + err => next(err), + ); + }, + next => { + s3.send(new AbortMultipartUploadCommand(params)) + .then(() => next()) + .catch(next); + }, + next => + setTimeout( + () => checkMPUList(gcpBucketMPU, this.test.key, this.test.uploadId, next), + gcpTimeout, + ), + ], + done, + ); }); }); describe('with previously existing object with same key', () => { beforeEach(function beforeEachFn(done) { - async.waterfall([ - next => { - s3.send(new CreateBucketCommand({ - Bucket: bucket, - })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: this.currentTest.key, - Metadata: { - 'scal-location-constraint': gcpLocation }, - Body: body, - })) - .then(() => next()) - .catch(err => { - assert.ifError(err, - `Expected success, got error: ${err}`); - next(err); - }); - }, - next => { - s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: this.currentTest.key, - Metadata: { 'scal-location-constraint': gcpLocation }, - })) - .then(res => { - this.currentTest.uploadId = res.UploadId; - next(); - }) - .catch(next); - }, - ], done); + async.waterfall( + [ + next => { + s3.send( + new CreateBucketCommand({ + Bucket: bucket, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: this.currentTest.key, + Metadata: { + 'scal-location-constraint': gcpLocation, + }, + Body: body, + }), + ) + .then(() => next()) + .catch(err => { + assert.ifError(err, `Expected success, got error: ${err}`); + next(err); + }); + }, + next => { + s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: this.currentTest.key, + Metadata: { 'scal-location-constraint': gcpLocation }, + }), + ) + .then(res => { + this.currentTest.uploadId = res.UploadId; + next(); + }) + .catch(next); + }, + ], + done, + ); }); afterEach(() => { process.stdout.write('Emptying bucket\n'); - return bucketUtil.empty(bucket) - .then(() => { - process.stdout.write('Deleting bucket\n'); - return bucketUtil.deleteOne(bucket); - }) - .catch(err => { - process.stdout.write('Error emptying/deleting bucket: ' + - `${err}\n`); - throw err; - }); + return bucketUtil + .empty(bucket) + .then(() => { + process.stdout.write('Deleting bucket\n'); + return bucketUtil.deleteOne(bucket); + }) + .catch(err => { + process.stdout.write('Error emptying/deleting bucket: ' + `${err}\n`); + throw err; + }); }); - it('should abort MPU without deleting existing object', - function itFn(done) { + it('should abort MPU without deleting existing object', function itFn(done) { const params = { Bucket: bucket, Key: this.test.key, UploadId: this.test.uploadId, }; - async.waterfall([ - next => { - const body = Buffer.alloc(10); - const partParams = Object.assign( - { PartNumber: 1, Body: body }, params); - s3.send(new UploadPartCommand(partParams)) - .then(() => next()) - .catch(err => { - assert.ifError(err, - `Expected success, got error: ${err}`); - next(err); - }); - }, - next => { - s3.send(new AbortMultipartUploadCommand(params)) - .then(() => next()) - .catch(next); - }, - next => setTimeout(() => { - const params = { - Bucket: gcpBucket, - Key: this.test.key, - }; - gcpClient.getObject(params, (err, res) => { - assert.ifError(err, - `Expected success, got error: ${err}`); - assert.strictEqual(res.ETag, `"${correctMD5}"`); - next(); - }); - }, gcpTimeout), - ], done); + async.waterfall( + [ + next => { + const body = Buffer.alloc(10); + const partParams = Object.assign({ PartNumber: 1, Body: body }, params); + s3.send(new UploadPartCommand(partParams)) + .then(() => next()) + .catch(err => { + assert.ifError(err, `Expected success, got error: ${err}`); + next(err); + }); + }, + next => { + s3.send(new AbortMultipartUploadCommand(params)) + .then(() => next()) + .catch(next); + }, + next => + setTimeout(() => { + const params = { + Bucket: gcpBucket, + Key: this.test.key, + }; + gcpClient.getObject(params, (err, res) => { + assert.ifError(err, `Expected success, got error: ${err}`); + assert.strictEqual(res.ETag, `"${correctMD5}"`); + next(); + }); + }, gcpTimeout), + ], + done, + ); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/mpuAbort/azureAbortMPU.js b/tests/functional/aws-node-sdk/test/multipleBackend/mpuAbort/azureAbortMPU.js index fe74499b07..741d7cba80 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/mpuAbort/azureAbortMPU.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/mpuAbort/azureAbortMPU.js @@ -12,9 +12,14 @@ const { const { s3middleware } = require('arsenal'); const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); -const { uniqName, getAzureClient, - getAzureContainerName, convertMD5, azureLocation, - describeSkipIfNotMultiple } = require('../utils'); +const { + uniqName, + getAzureClient, + getAzureContainerName, + convertMD5, + azureLocation, + describeSkipIfNotMultiple, +} = require('../utils'); const azureMpuUtils = s3middleware.azureHelper.mpuUtils; const maxSubPartSize = azureMpuUtils.maxSubPartSize; @@ -27,22 +32,26 @@ let bucketUtil; let s3; function azureCheck(container, key, expected, cb) { - azureClient.getContainerClient(container).getProperties(key).then(res => { - assert.ok(!expected.error); - const convertedMD5 = convertMD5(res.contentSettings.contentMD5); - assert.strictEqual(convertedMD5, expectedMD5); - return cb(); - }, - err => { - assert.ok(expected.error); - assert.strictEqual(err.statusCode, 404); - assert.strictEqual(err.code, 'NotFound'); - return cb(); - }); + azureClient + .getContainerClient(container) + .getProperties(key) + .then( + res => { + assert.ok(!expected.error); + const convertedMD5 = convertMD5(res.contentSettings.contentMD5); + assert.strictEqual(convertedMD5, expectedMD5); + return cb(); + }, + err => { + assert.ok(expected.error); + assert.strictEqual(err.statusCode, 404); + assert.strictEqual(err.code, 'NotFound'); + return cb(); + }, + ); } -describeSkipIfNotMultiple('Abort MPU on Azure data backend', function -describeF() { +describeSkipIfNotMultiple('Abort MPU on Azure data backend', function describeF() { this.timeout(50000); withV4(sigCfg => { beforeEach(function beforeFn() { @@ -52,35 +61,44 @@ describeF() { }); describe('with bucket location header', () => { beforeEach(function beforeEachFn(done) { - async.waterfall([ - next => { - s3.send(new CreateBucketCommand({ - Bucket: azureContainerName, - })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new CreateMultipartUploadCommand({ - Bucket: azureContainerName, - Key: this.currentTest.key, - Metadata: { - 'scal-location-constraint': azureLocation, - }, - })) - .then(res => { - this.currentTest.uploadId = res.UploadId; - next(); - }) - .catch(next); - }, - ], done); + async.waterfall( + [ + next => { + s3.send( + new CreateBucketCommand({ + Bucket: azureContainerName, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + s3.send( + new CreateMultipartUploadCommand({ + Bucket: azureContainerName, + Key: this.currentTest.key, + Metadata: { + 'scal-location-constraint': azureLocation, + }, + }), + ) + .then(res => { + this.currentTest.uploadId = res.UploadId; + next(); + }) + .catch(next); + }, + ], + done, + ); }); afterEach(done => { - s3.send(new DeleteBucketCommand({ - Bucket: azureContainerName, - })) + s3.send( + new DeleteBucketCommand({ + Bucket: azureContainerName, + }), + ) .then(() => done()) .catch(done); }); @@ -92,144 +110,156 @@ describeF() { Key: this.test.key, UploadId: this.test.uploadId, }; - async.waterfall([ - next => { - const partParams = { - ...params, - PartNumber: 1, - Body: Buffer.alloc(0), - }; - s3.send(new UploadPartCommand(partParams)) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new AbortMultipartUploadCommand(params)) - .then(() => next()) - .catch(next); - }, - next => azureCheck(azureContainerName, this.test.key, - expected, next), - ], done); + async.waterfall( + [ + next => { + const partParams = { + ...params, + PartNumber: 1, + Body: Buffer.alloc(0), + }; + s3.send(new UploadPartCommand(partParams)) + .then(() => next()) + .catch(next); + }, + next => { + s3.send(new AbortMultipartUploadCommand(params)) + .then(() => next()) + .catch(next); + }, + next => azureCheck(azureContainerName, this.test.key, expected, next), + ], + done, + ); }); - it('should abort MPU with one part bigger than max subpart', - function itFn(done) { + it('should abort MPU with one part bigger than max subpart', function itFn(done) { const expected = { error: true }; const params = { Bucket: azureContainerName, Key: this.test.key, UploadId: this.test.uploadId, }; - async.waterfall([ - next => { - const body = Buffer.alloc(maxSubPartSize + 10); - const partParams = { - ...params, - PartNumber: 1, - Body: body, - }; - s3.send(new UploadPartCommand(partParams)) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new AbortMultipartUploadCommand(params)) - .then(() => next()) - .catch(next); - }, - next => azureCheck(azureContainerName, this.test.key, - expected, next), - ], done); + async.waterfall( + [ + next => { + const body = Buffer.alloc(maxSubPartSize + 10); + const partParams = { + ...params, + PartNumber: 1, + Body: body, + }; + s3.send(new UploadPartCommand(partParams)) + .then(() => next()) + .catch(next); + }, + next => { + s3.send(new AbortMultipartUploadCommand(params)) + .then(() => next()) + .catch(next); + }, + next => azureCheck(azureContainerName, this.test.key, expected, next), + ], + done, + ); }); }); describe('with previously existing object with same key', () => { beforeEach(function beforeEachFn(done) { - async.waterfall([ - next => { - s3.send(new CreateBucketCommand({ - Bucket: azureContainerName, - })) - .then(() => next()) - .catch(next); - }, - next => { - const body = Buffer.alloc(10); - s3.send(new PutObjectCommand({ - Bucket: azureContainerName, - Key: this.currentTest.key, - Metadata: { - 'scal-location-constraint': azureLocation, - }, - Body: body, - })) - .then(() => next()) - .catch(err => { - assert.equal(err, null, 'Err putting object to ' + - `azure: ${err}`); - next(err); - }); - }, - next => { - s3.send(new CreateMultipartUploadCommand({ - Bucket: azureContainerName, - Key: this.currentTest.key, - Metadata: { - 'scal-location-constraint': azureLocation, - }, - })) - .then(res => { - this.currentTest.uploadId = res.UploadId; - next(); - }) - .catch(next); - }, - ], done); + async.waterfall( + [ + next => { + s3.send( + new CreateBucketCommand({ + Bucket: azureContainerName, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + const body = Buffer.alloc(10); + s3.send( + new PutObjectCommand({ + Bucket: azureContainerName, + Key: this.currentTest.key, + Metadata: { + 'scal-location-constraint': azureLocation, + }, + Body: body, + }), + ) + .then(() => next()) + .catch(err => { + assert.equal(err, null, 'Err putting object to ' + `azure: ${err}`); + next(err); + }); + }, + next => { + s3.send( + new CreateMultipartUploadCommand({ + Bucket: azureContainerName, + Key: this.currentTest.key, + Metadata: { + 'scal-location-constraint': azureLocation, + }, + }), + ) + .then(res => { + this.currentTest.uploadId = res.UploadId; + next(); + }) + .catch(next); + }, + ], + done, + ); }); afterEach(() => { process.stdout.write('Emptying bucket\n'); - return bucketUtil.empty(azureContainerName) - .then(() => { - process.stdout.write('Deleting bucket\n'); - return bucketUtil.deleteOne(azureContainerName); - }) - .catch(err => { - process.stdout.write('Error emptying/deleting bucket: ' + - `${err}\n`); - throw err; - }); + return bucketUtil + .empty(azureContainerName) + .then(() => { + process.stdout.write('Deleting bucket\n'); + return bucketUtil.deleteOne(azureContainerName); + }) + .catch(err => { + process.stdout.write('Error emptying/deleting bucket: ' + `${err}\n`); + throw err; + }); }); - it('should abort MPU without deleting existing object', - function itFn(done) { + it('should abort MPU without deleting existing object', function itFn(done) { const expected = { error: false }; const params = { Bucket: azureContainerName, Key: this.test.key, UploadId: this.test.uploadId, }; - async.waterfall([ - next => { - const body = Buffer.alloc(10); - const partParams = { - ...params, - PartNumber: 1, - Body: body, - }; - s3.send(new UploadPartCommand(partParams)) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new AbortMultipartUploadCommand(params)) - .then(() => next()) - .catch(next); - }, - next => azureCheck(azureContainerName, this.test.key, - expected, next), - ], done); + async.waterfall( + [ + next => { + const body = Buffer.alloc(10); + const partParams = { + ...params, + PartNumber: 1, + Body: body, + }; + s3.send(new UploadPartCommand(partParams)) + .then(() => next()) + .catch(next); + }, + next => { + s3.send(new AbortMultipartUploadCommand(params)) + .then(() => next()) + .catch(next); + }, + next => azureCheck(azureContainerName, this.test.key, expected, next), + ], + done, + ); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/mpuComplete/azureCompleteMPU.js b/tests/functional/aws-node-sdk/test/multipleBackend/mpuComplete/azureCompleteMPU.js index 19e17e48b7..40841a0994 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/mpuComplete/azureCompleteMPU.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/mpuComplete/azureCompleteMPU.js @@ -1,11 +1,13 @@ const async = require('async'); const assert = require('assert'); -const { CreateBucketCommand, +const { + CreateBucketCommand, CreateMultipartUploadCommand, UploadPartCommand, CompleteMultipartUploadCommand, PutObjectCommand, - GetObjectCommand } = require('@aws-sdk/client-s3'); + GetObjectCommand, +} = require('@aws-sdk/client-s3'); const { s3middleware } = require('arsenal'); const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); @@ -58,64 +60,66 @@ function getCheck(key, bucketMatch, cb) { } function mpuSetup(key, location, cb) { const partArray = []; - async.waterfall([ - next => { - const params = { - Bucket: azureContainerName, - Key: key, - Metadata: { 'scal-location-constraint': location }, - }; - s3.send(new CreateMultipartUploadCommand(params)) - .then(res => { - const uploadId = res.UploadId; - assert(uploadId); - assert.strictEqual(res.Bucket, azureContainerName); - assert.strictEqual(res.Key, key); - return next(null, uploadId); - }) - .catch(next); - }, - (uploadId, next) => { - const partParams = { - Bucket: azureContainerName, - Key: key, - PartNumber: 1, - UploadId: uploadId, - Body: smallBody, - }; - s3.send(new UploadPartCommand(partParams)) - .then(res => { - partArray.push({ ETag: res.ETag, PartNumber: 1 }); - return next(null, uploadId); - }) - .catch(next); - }, - (uploadId, next) => { - const partParams = { - Bucket: azureContainerName, - Key: key, - PartNumber: 2, - UploadId: uploadId, - Body: bigBody, - }; - s3.send(new UploadPartCommand(partParams)) - .then(res => { - partArray.push({ ETag: res.ETag, PartNumber: 2 }); - return next(null, uploadId); - }) - .catch(next); + async.waterfall( + [ + next => { + const params = { + Bucket: azureContainerName, + Key: key, + Metadata: { 'scal-location-constraint': location }, + }; + s3.send(new CreateMultipartUploadCommand(params)) + .then(res => { + const uploadId = res.UploadId; + assert(uploadId); + assert.strictEqual(res.Bucket, azureContainerName); + assert.strictEqual(res.Key, key); + return next(null, uploadId); + }) + .catch(next); + }, + (uploadId, next) => { + const partParams = { + Bucket: azureContainerName, + Key: key, + PartNumber: 1, + UploadId: uploadId, + Body: smallBody, + }; + s3.send(new UploadPartCommand(partParams)) + .then(res => { + partArray.push({ ETag: res.ETag, PartNumber: 1 }); + return next(null, uploadId); + }) + .catch(next); + }, + (uploadId, next) => { + const partParams = { + Bucket: azureContainerName, + Key: key, + PartNumber: 2, + UploadId: uploadId, + Body: bigBody, + }; + s3.send(new UploadPartCommand(partParams)) + .then(res => { + partArray.push({ ETag: res.ETag, PartNumber: 2 }); + return next(null, uploadId); + }) + .catch(next); + }, + ], + (err, uploadId) => { + if (err) { + return cb(err); + } + process.stdout.write('Created MPU and put two parts\n'); + return cb(uploadId, partArray); }, - ], (err, uploadId) => { - if (err) { - return cb(err); - } - process.stdout.write('Created MPU and put two parts\n'); - return cb(uploadId, partArray); - }); + ); } -describeSkipIfNotMultiple('Complete MPU API for Azure data backend', -function testSuite() { +describeSkipIfNotMultiple('Complete MPU API for Azure data backend', function testSuite() { this.timeout(150000); withV4(sigCfg => { beforeEach(function beFn() { @@ -123,8 +127,7 @@ function testSuite() { bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; this.currentTest.awsClient = awsS3; - return s3.send(new CreateBucketCommand({ Bucket: azureContainerName })) - .catch(err => { + return s3.send(new CreateBucketCommand({ Bucket: azureContainerName })).catch(err => { process.stdout.write(`Error creating bucket: ${err}\n`); throw err; }); @@ -132,15 +135,16 @@ function testSuite() { afterEach(() => { process.stdout.write('Emptying bucket\n'); - return bucketUtil.empty(azureContainerName) - .then(() => { - process.stdout.write('Deleting bucket\n'); - return bucketUtil.deleteOne(azureContainerName); - }) - .catch(err => { - process.stdout.write(`Error in afterEach: ${err}\n`); - throw err; - }); + return bucketUtil + .empty(azureContainerName) + .then(() => { + process.stdout.write('Deleting bucket\n'); + return bucketUtil.deleteOne(azureContainerName); + }) + .catch(err => { + process.stdout.write(`Error in afterEach: ${err}\n`); + throw err; + }); }); it('should complete an MPU on Azure', function itFn(done) { @@ -153,17 +157,14 @@ function testSuite() { }; s3.send(new CompleteMultipartUploadCommand(params)) .then(() => { - setTimeout(() => getCheck(this.test.key, true, done), - azureTimeout); + setTimeout(() => getCheck(this.test.key, true, done), azureTimeout); }) .catch(done); }); }); - it('should complete an MPU on Azure with bucketMatch=false', - function itFn(done) { - mpuSetup(this.test.key, azureLocationMismatch, - (uploadId, partArray) => { + it('should complete an MPU on Azure with bucketMatch=false', function itFn(done) { + mpuSetup(this.test.key, azureLocationMismatch, (uploadId, partArray) => { const params = { Bucket: azureContainerName, Key: this.test.key, @@ -172,79 +173,81 @@ function testSuite() { }; s3.send(new CompleteMultipartUploadCommand(params)) .then(() => { - setTimeout(() => getCheck(this.test.key, false, done), - azureTimeout); + setTimeout(() => getCheck(this.test.key, false, done), azureTimeout); }) .catch(done); }); }); - it('should complete an MPU on Azure with same key as object put ' + - 'to file', function itFn(done) { + it('should complete an MPU on Azure with same key as object put ' + 'to file', function itFn(done) { const body = Buffer.from('I am a body', 'utf8'); - s3.send(new PutObjectCommand({ - Bucket: azureContainerName, - Key: this.test.key, - Body: body, - Metadata: { 'scal-location-constraint': fileLocation } })).then(() => { - mpuSetup(this.test.key, azureLocation, - (uploadId, partArray) => { - const params = { - Bucket: azureContainerName, - Key: this.test.key, - UploadId: uploadId, - MultipartUpload: { Parts: partArray }, - }; - s3.send(new CompleteMultipartUploadCommand(params)) - .then(() => { - setTimeout(() => getCheck(this.test.key, true, done), - azureTimeout); - }) - .catch(done); - }); - }).catch(done); + s3.send( + new PutObjectCommand({ + Bucket: azureContainerName, + Key: this.test.key, + Body: body, + Metadata: { 'scal-location-constraint': fileLocation }, + }), + ) + .then(() => { + mpuSetup(this.test.key, azureLocation, (uploadId, partArray) => { + const params = { + Bucket: azureContainerName, + Key: this.test.key, + UploadId: uploadId, + MultipartUpload: { Parts: partArray }, + }; + s3.send(new CompleteMultipartUploadCommand(params)) + .then(() => { + setTimeout(() => getCheck(this.test.key, true, done), azureTimeout); + }) + .catch(done); + }); + }) + .catch(done); }); - it('should complete an MPU on Azure with same key as object put ' + - 'to Azure', function itFn(done) { + it('should complete an MPU on Azure with same key as object put ' + 'to Azure', function itFn(done) { const body = Buffer.from('I am a body', 'utf8'); - s3.send(new PutObjectCommand({ - Bucket: azureContainerName, - Key: this.test.key, - Body: body, - Metadata: { 'scal-location-constraint': azureLocation } })).then(() => { - mpuSetup(this.test.key, azureLocation, - (uploadId, partArray) => { + s3.send( + new PutObjectCommand({ + Bucket: azureContainerName, + Key: this.test.key, + Body: body, + Metadata: { 'scal-location-constraint': azureLocation }, + }), + ).then(() => { + mpuSetup(this.test.key, azureLocation, (uploadId, partArray) => { const params = { Bucket: azureContainerName, Key: this.test.key, UploadId: uploadId, MultipartUpload: { Parts: partArray }, }; - s3.send(new CompleteMultipartUploadCommand(params)).then(() => { - - setTimeout(() => getCheck(this.test.key, true, done), - azureTimeout); - }).catch(err => { - assert.equal(err, null, `Err completing MPU: ${err}`); - done(err); - }); + s3.send(new CompleteMultipartUploadCommand(params)) + .then(() => { + setTimeout(() => getCheck(this.test.key, true, done), azureTimeout); + }) + .catch(err => { + assert.equal(err, null, `Err completing MPU: ${err}`); + done(err); + }); }); }); }); - it('should complete an MPU on Azure with same key as object put ' + - 'to AWS', function itFn(done) { + it('should complete an MPU on Azure with same key as object put ' + 'to AWS', function itFn(done) { const body = Buffer.from('I am a body', 'utf8'); - s3.send(new PutObjectCommand({ - Bucket: azureContainerName, - Key: this.test.key, - Body: body, - Metadata: { 'scal-location-constraint': awsLocation } - })) + s3.send( + new PutObjectCommand({ + Bucket: azureContainerName, + Key: this.test.key, + Body: body, + Metadata: { 'scal-location-constraint': awsLocation }, + }), + ) .then(() => { - mpuSetup(this.test.key, azureLocation, - (uploadId, partArray) => { + mpuSetup(this.test.key, azureLocation, (uploadId, partArray) => { const params = { Bucket: azureContainerName, Key: this.test.key, @@ -255,10 +258,13 @@ function testSuite() { .then(() => { // make sure object is gone from AWS setTimeout(() => { - this.test.awsClient.send(new GetObjectCommand({ - Bucket: awsBucket, - Key: this.test.key - })) + this.test.awsClient + .send( + new GetObjectCommand({ + Bucket: awsBucket, + Key: this.test.key, + }), + ) .then(() => { done(new Error('Expected NoSuchKey error')); }) diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/mpuComplete/completeMPUGcp.js b/tests/functional/aws-node-sdk/test/multipleBackend/mpuComplete/completeMPUGcp.js index 0b346fb4cb..624a590c40 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/mpuComplete/completeMPUGcp.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/mpuComplete/completeMPUGcp.js @@ -11,9 +11,18 @@ const { const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); -const { fileLocation, awsS3, awsLocation, - awsBucket, gcpClient, gcpBucket, gcpLocation, gcpLocationMismatch, - genUniqID, describeSkipIfNotMultiple } = require('../utils'); +const { + fileLocation, + awsS3, + awsLocation, + awsBucket, + gcpClient, + gcpBucket, + gcpLocation, + gcpLocationMismatch, + genUniqID, + describeSkipIfNotMultiple, +} = require('../utils'); const bucket = `completempugcp${genUniqID()}`; const smallBody = Buffer.from('I am a body', 'utf8'); @@ -28,10 +37,12 @@ let bucketUtil; function getCheck(key, bucketMatch, cb) { (async () => { let gcpKey = key; - const s3Res = await s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: gcpKey, - })); + const s3Res = await s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: gcpKey, + }), + ); assert.strictEqual(s3Res.ETag, `"${s3MD5}"`); if (!bucketMatch) { @@ -52,59 +63,67 @@ function getCheck(key, bucketMatch, cb) { function mpuSetup(key, location, cb) { const partArray = []; - async.waterfall([ - next => { - s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: key, - Metadata: { 'scal-location-constraint': location }, - })) - .then(res => { - const uploadId = res.UploadId; - assert(uploadId); - assert.strictEqual(res.Bucket, bucket); - assert.strictEqual(res.Key, key); - next(null, uploadId); - }) - .catch(next); + async.waterfall( + [ + next => { + s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + Metadata: { 'scal-location-constraint': location }, + }), + ) + .then(res => { + const uploadId = res.UploadId; + assert(uploadId); + assert.strictEqual(res.Bucket, bucket); + assert.strictEqual(res.Key, key); + next(null, uploadId); + }) + .catch(next); + }, + (uploadId, next) => { + s3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: key, + PartNumber: 1, + UploadId: uploadId, + Body: smallBody, + }), + ) + .then(res => { + partArray.push({ ETag: res.ETag, PartNumber: 1 }); + next(null, uploadId); + }) + .catch(next); + }, + (uploadId, next) => { + s3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: key, + PartNumber: 2, + UploadId: uploadId, + Body: bigBody, + }), + ) + .then(res => { + partArray.push({ ETag: res.ETag, PartNumber: 2 }); + next(null, uploadId); + }) + .catch(next); + }, + ], + (err, uploadId) => { + process.stdout.write('Created MPU and put two parts\n'); + assert.equal(err, null, `Err setting up MPU: ${err}`); + cb(uploadId, partArray); }, - (uploadId, next) => { - s3.send(new UploadPartCommand({ - Bucket: bucket, - Key: key, - PartNumber: 1, - UploadId: uploadId, - Body: smallBody, - })) - .then(res => { - partArray.push({ ETag: res.ETag, PartNumber: 1 }); - next(null, uploadId); - }) - .catch(next); - }, - (uploadId, next) => { - s3.send(new UploadPartCommand({ - Bucket: bucket, - Key: key, - PartNumber: 2, - UploadId: uploadId, - Body: bigBody, - })) - .then(res => { - partArray.push({ ETag: res.ETag, PartNumber: 2 }); - next(null, uploadId); - }) - .catch(next); - }, - ], (err, uploadId) => { - process.stdout.write('Created MPU and put two parts\n'); - assert.equal(err, null, `Err setting up MPU: ${err}`); - cb(uploadId, partArray); - }); + ); } -describeSkipIfNotMultiple('Complete MPU API for GCP data backend', -function testSuite() { +describeSkipIfNotMultiple('Complete MPU API for GCP data backend', function testSuite() { this.timeout(150000); withV4(sigCfg => { beforeEach(function beFn() { @@ -112,24 +131,24 @@ function testSuite() { bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; this.currentTest.awsClient = awsS3; - return s3.send(new CreateBucketCommand({ Bucket: bucket })) - .catch(err => { - process.stdout.write(`Error creating bucket: ${err}\n`); - throw err; - }); + return s3.send(new CreateBucketCommand({ Bucket: bucket })).catch(err => { + process.stdout.write(`Error creating bucket: ${err}\n`); + throw err; + }); }); afterEach(() => { process.stdout.write('Emptying bucket\n'); - return bucketUtil.empty(bucket) - .then(() => { - process.stdout.write('Deleting bucket\n'); - return bucketUtil.deleteOne(bucket); - }) - .catch(err => { - process.stdout.write(`Error in afterEach: ${err}\n`); - throw err; - }); + return bucketUtil + .empty(bucket) + .then(() => { + process.stdout.write('Deleting bucket\n'); + return bucketUtil.deleteOne(bucket); + }) + .catch(err => { + process.stdout.write(`Error in afterEach: ${err}\n`); + throw err; + }); }); it('should complete an MPU on GCP', function itFn(done) { @@ -148,10 +167,8 @@ function testSuite() { }); }); - it('should complete an MPU on GCP with bucketMatch=false', - function itFn(done) { - mpuSetup(this.test.key, gcpLocationMismatch, - (uploadId, partArray) => { + it('should complete an MPU on GCP with bucketMatch=false', function itFn(done) { + mpuSetup(this.test.key, gcpLocationMismatch, (uploadId, partArray) => { const params = { Bucket: bucket, Key: this.test.key, @@ -161,23 +178,23 @@ function testSuite() { setTimeout(() => { s3.send(new CompleteMultipartUploadCommand(params)) .then(() => getCheck(this.test.key, false, done)) - .catch(done); + .catch(done); }, gcpTimeout); }); }); - it('should complete an MPU on GCP with same key as object put ' + - 'to file', function itFn(done) { + it('should complete an MPU on GCP with same key as object put ' + 'to file', function itFn(done) { const body = Buffer.from('I am a body', 'utf8'); - s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: this.test.key, - Body: body, - Metadata: { 'scal-location-constraint': fileLocation }, - })) + s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: this.test.key, + Body: body, + Metadata: { 'scal-location-constraint': fileLocation }, + }), + ) .then(() => { - mpuSetup(this.test.key, gcpLocation, - (uploadId, partArray) => { + mpuSetup(this.test.key, gcpLocation, (uploadId, partArray) => { const params = { Bucket: bucket, Key: this.test.key, @@ -197,18 +214,18 @@ function testSuite() { }); }); - it('should complete an MPU on GCP with same key as object put ' + - 'to GCP', function itFn(done) { + it('should complete an MPU on GCP with same key as object put ' + 'to GCP', function itFn(done) { const body = Buffer.from('I am a body', 'utf8'); - s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: this.test.key, - Body: body, - Metadata: { 'scal-location-constraint': gcpLocation }, - })) + s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: this.test.key, + Body: body, + Metadata: { 'scal-location-constraint': gcpLocation }, + }), + ) .then(() => { - mpuSetup(this.test.key, gcpLocation, - (uploadId, partArray) => { + mpuSetup(this.test.key, gcpLocation, (uploadId, partArray) => { const params = { Bucket: bucket, Key: this.test.key, @@ -228,18 +245,18 @@ function testSuite() { }); }); - it('should complete an MPU on GCP with same key as object put ' + - 'to AWS', function itFn(done) { + it('should complete an MPU on GCP with same key as object put ' + 'to AWS', function itFn(done) { const body = Buffer.from('I am a body', 'utf8'); - s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: this.test.key, - Body: body, - Metadata: { 'scal-location-constraint': awsLocation }, - })) + s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: this.test.key, + Body: body, + Metadata: { 'scal-location-constraint': awsLocation }, + }), + ) .then(() => { - mpuSetup(this.test.key, gcpLocation, - (uploadId, partArray) => { + mpuSetup(this.test.key, gcpLocation, (uploadId, partArray) => { const params = { Bucket: bucket, Key: this.test.key, @@ -250,8 +267,7 @@ function testSuite() { .then(() => { // make sure object is gone from AWS setTimeout(() => { - this.test.awsClient.getObject({ Bucket: awsBucket, - Key: this.test.key }, err => { + this.test.awsClient.getObject({ Bucket: awsBucket, Key: this.test.key }, err => { assert.strictEqual(err.code, 'NoSuchKey'); getCheck(this.test.key, true, done); }); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/mpuComplete/mpuAwsVersioning.js b/tests/functional/aws-node-sdk/test/multipleBackend/mpuComplete/mpuAwsVersioning.js index 766208c85f..927ac88a7c 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/mpuComplete/mpuAwsVersioning.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/mpuComplete/mpuAwsVersioning.js @@ -2,9 +2,14 @@ const assert = require('assert'); const async = require('async'); const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); -const { CreateBucketCommand, DeleteBucketCommand, - CreateMultipartUploadCommand, UploadPartCommand, - CompleteMultipartUploadCommand, DeleteObjectCommand } = require('@aws-sdk/client-s3'); +const { + CreateBucketCommand, + DeleteBucketCommand, + CreateMultipartUploadCommand, + UploadPartCommand, + CompleteMultipartUploadCommand, + DeleteObjectCommand, +} = require('@aws-sdk/client-s3'); const { minimumAllowedPartSize } = require('../../../../../../constants'); const { removeAllVersions } = require('../../../lib/utility/versioning-util'); const { @@ -25,161 +30,195 @@ const bucket = `mpuawsversioning${genUniqID()}`; function mpuSetup(s3, key, location, cb) { const partArray = []; - async.waterfall([ - next => { - const params = { - Bucket: bucket, - Key: key, - Metadata: { 'scal-location-constraint': location }, - }; - s3.send(new CreateMultipartUploadCommand(params)).then(res => { - const uploadId = res.UploadId; - assert(uploadId); - assert.strictEqual(res.Bucket, bucket); - assert.strictEqual(res.Key, key); - next(null, uploadId); - }).catch(err => { - next(err); - }); - }, - (uploadId, next) => { - const partParams = { - Bucket: bucket, - Key: key, - PartNumber: 1, - UploadId: uploadId, - Body: data[0], - }; - s3.send(new UploadPartCommand(partParams)).then(res => { - partArray.push({ ETag: res.ETag, PartNumber: 1 }); - next(null, uploadId); - }).catch(err => { - next(err); - }); - }, - (uploadId, next) => { - const partParams = { - Bucket: bucket, - Key: key, - PartNumber: 2, - UploadId: uploadId, - Body: data[1], - }; - s3.send(new UploadPartCommand(partParams)).then(res => { - partArray.push({ ETag: res.ETag, PartNumber: 2 }); - next(null, uploadId); - }).catch(err => { - next(err); - }); + async.waterfall( + [ + next => { + const params = { + Bucket: bucket, + Key: key, + Metadata: { 'scal-location-constraint': location }, + }; + s3.send(new CreateMultipartUploadCommand(params)) + .then(res => { + const uploadId = res.UploadId; + assert(uploadId); + assert.strictEqual(res.Bucket, bucket); + assert.strictEqual(res.Key, key); + next(null, uploadId); + }) + .catch(err => { + next(err); + }); + }, + (uploadId, next) => { + const partParams = { + Bucket: bucket, + Key: key, + PartNumber: 1, + UploadId: uploadId, + Body: data[0], + }; + s3.send(new UploadPartCommand(partParams)) + .then(res => { + partArray.push({ ETag: res.ETag, PartNumber: 1 }); + next(null, uploadId); + }) + .catch(err => { + next(err); + }); + }, + (uploadId, next) => { + const partParams = { + Bucket: bucket, + Key: key, + PartNumber: 2, + UploadId: uploadId, + Body: data[1], + }; + s3.send(new UploadPartCommand(partParams)) + .then(res => { + partArray.push({ ETag: res.ETag, PartNumber: 2 }); + next(null, uploadId); + }) + .catch(err => { + next(err); + }); + }, + ], + (err, uploadId) => { + process.stdout.write('Created MPU and put two parts\n'); + cb(err, uploadId, partArray); }, - ], (err, uploadId) => { - process.stdout.write('Created MPU and put two parts\n'); - cb(err, uploadId, partArray); - }); + ); } function completeAndAssertMpu(s3, params, cb) { - const { bucket, key, uploadId, partArray, expectVersionId, - expectedGetVersionId } = params; - s3.send(new CompleteMultipartUploadCommand({ - Bucket: bucket, - Key: key, - UploadId: uploadId, - MultipartUpload: { Parts: partArray }, - })).then(data => { - if (expectVersionId) { - assert.notEqual(data.VersionId, undefined); - } else { - assert.strictEqual(data.VersionId, undefined); - } - const expectedVersionId = expectedGetVersionId || data.VersionId; - getAndAssertResult(s3, { bucket, key, body: concattedData, - expectedVersionId }, cb); - }).catch(err => { - cb(err); - }); + const { bucket, key, uploadId, partArray, expectVersionId, expectedGetVersionId } = params; + s3.send( + new CompleteMultipartUploadCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + MultipartUpload: { Parts: partArray }, + }), + ) + .then(data => { + if (expectVersionId) { + assert.notEqual(data.VersionId, undefined); + } else { + assert.strictEqual(data.VersionId, undefined); + } + const expectedVersionId = expectedGetVersionId || data.VersionId; + getAndAssertResult(s3, { bucket, key, body: concattedData, expectedVersionId }, cb); + }) + .catch(err => { + cb(err); + }); } -describeSkipIfNotMultiple('AWS backend complete mpu with versioning', -function testSuite() { +describeSkipIfNotMultiple('AWS backend complete mpu with versioning', function testSuite() { this.timeout(120000); withV4(sigCfg => { const bucketUtil = new BucketUtility('default', sigCfg); const s3 = bucketUtil.s3; - beforeEach(done => s3.send(new CreateBucketCommand({ - Bucket: bucket, - CreateBucketConfiguration: { - LocationConstraint: awsLocation, - }, - })).then(() => done()).catch(err => done(err))); + beforeEach(done => + s3 + .send( + new CreateBucketCommand({ + Bucket: bucket, + CreateBucketConfiguration: { + LocationConstraint: awsLocation, + }, + }), + ) + .then(() => done()) + .catch(err => done(err)), + ); afterEach(done => { removeAllVersions({ Bucket: bucket }, err => { if (err) { return done(err); } - return s3.send(new DeleteBucketCommand({ Bucket: bucket })) - .then(() => done()).catch(done); + return s3 + .send(new DeleteBucketCommand({ Bucket: bucket })) + .then(() => done()) + .catch(done); }); }); - it('versioning not configured: should not return version id ' + - 'completing mpu', done => { + it('versioning not configured: should not return version id ' + 'completing mpu', done => { const key = `somekey-${genUniqID()}`; mpuSetup(s3, key, awsLocation, (err, uploadId, partArray) => { - completeAndAssertMpu(s3, { bucket, key, uploadId, partArray, - expectVersionId: false }, done); + completeAndAssertMpu(s3, { bucket, key, uploadId, partArray, expectVersionId: false }, done); }); }); - it('versioning not configured: if complete mpu on already-existing ' + - 'object, metadata should be overwritten but data of previous version' + - 'in AWS should not be deleted', function itF(done) { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => putToAwsBackend(s3, bucket, key, '', err => next(err)), - next => awsGetLatestVerId(key, '', next), - (awsVerId, next) => { - this.test.awsVerId = awsVerId; - next(); - }, - next => mpuSetup(s3, key, awsLocation, next), - (uploadId, partArray, next) => completeAndAssertMpu(s3, - { bucket, key, uploadId, partArray, expectVersionId: - false }, next), - next => s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: key, VersionId: - 'null' })).then(delData => next(null, delData)).catch(next), - (delData, next) => getAndAssertResult(s3, { bucket, key, - expectedError: 'NoSuchKey' }, next), - next => awsGetLatestVerId(key, '', next), - (awsVerId, next) => { - assert.strictEqual(awsVerId, this.test.awsVerId); - next(); - }, - ], done); - }); + it( + 'versioning not configured: if complete mpu on already-existing ' + + 'object, metadata should be overwritten but data of previous version' + + 'in AWS should not be deleted', + function itF(done) { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => putToAwsBackend(s3, bucket, key, '', err => next(err)), + next => awsGetLatestVerId(key, '', next), + (awsVerId, next) => { + this.test.awsVerId = awsVerId; + next(); + }, + next => mpuSetup(s3, key, awsLocation, next), + (uploadId, partArray, next) => + completeAndAssertMpu( + s3, + { bucket, key, uploadId, partArray, expectVersionId: false }, + next, + ), + next => + s3 + .send(new DeleteObjectCommand({ Bucket: bucket, Key: key, VersionId: 'null' })) + .then(delData => next(null, delData)) + .catch(next), + (delData, next) => getAndAssertResult(s3, { bucket, key, expectedError: 'NoSuchKey' }, next), + next => awsGetLatestVerId(key, '', next), + (awsVerId, next) => { + assert.strictEqual(awsVerId, this.test.awsVerId); + next(); + }, + ], + done, + ); + }, + ); - it('versioning suspended: should not return version id completing mpu', - done => { + it('versioning suspended: should not return version id completing mpu', done => { const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => suspendVersioning(s3, bucket, next), - next => mpuSetup(s3, key, awsLocation, next), - (uploadId, partArray, next) => completeAndAssertMpu(s3, - { bucket, key, uploadId, partArray, expectVersionId: false, - expectedGetVersionId: 'null' }, next), - ], done); + async.waterfall( + [ + next => suspendVersioning(s3, bucket, next), + next => mpuSetup(s3, key, awsLocation, next), + (uploadId, partArray, next) => + completeAndAssertMpu( + s3, + { bucket, key, uploadId, partArray, expectVersionId: false, expectedGetVersionId: 'null' }, + next, + ), + ], + done, + ); }); - it('versioning enabled: should return version id completing mpu', - done => { + it('versioning enabled: should return version id completing mpu', done => { const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => enableVersioning(s3, bucket, next), - next => mpuSetup(s3, key, awsLocation, next), - (uploadId, partArray, next) => completeAndAssertMpu(s3, - { bucket, key, uploadId, partArray, expectVersionId: true }, - next), - ], done); + async.waterfall( + [ + next => enableVersioning(s3, bucket, next), + next => mpuSetup(s3, key, awsLocation, next), + (uploadId, partArray, next) => + completeAndAssertMpu(s3, { bucket, key, uploadId, partArray, expectVersionId: true }, next), + ], + done, + ); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/mpuParts/azurePutPart.js b/tests/functional/aws-node-sdk/test/multipleBackend/mpuParts/azurePutPart.js index f342472f90..4d80bfad68 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/mpuParts/azurePutPart.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/mpuParts/azurePutPart.js @@ -14,10 +14,16 @@ const { const { s3middleware } = require('arsenal'); const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); -const { expectedETag, uniqName, getAzureClient, - getAzureContainerName, convertMD5, azureLocation, azureLocationMismatch, - describeSkipIfNotMultiple } - = require('../utils'); +const { + expectedETag, + uniqName, + getAzureClient, + getAzureContainerName, + convertMD5, + azureLocation, + azureLocationMismatch, + describeSkipIfNotMultiple, +} = require('../utils'); const azureMpuUtils = s3middleware.azureHelper.mpuUtils; const maxSubPartSize = azureMpuUtils.maxSubPartSize; const getBlockId = azureMpuUtils.getBlockId; @@ -31,15 +37,19 @@ let bucketUtil; let s3; function checkSubPart(key, uploadId, expectedParts, cb) { - azureClient.getContainerClient(azureContainerName) + azureClient + .getContainerClient(azureContainerName) .getBlockBlobClient(key) - .getBlockList('all').then(list => { + .getBlockList('all') + .then(list => { const uncommittedBlocks = list.uncommittedBlocks; const committedBlocks = list.committedBlocks; assert.strictEqual(committedBlocks, undefined); uncommittedBlocks.forEach((l, index) => { - assert.strictEqual(l.name, getBlockId(uploadId, - expectedParts[index].partnbr, expectedParts[index].subpartnbr)); + assert.strictEqual( + l.name, + getBlockId(uploadId, expectedParts[index].partnbr, expectedParts[index].subpartnbr), + ); assert.strictEqual(l.size, expectedParts[index].size.toString()); }); cb(); @@ -48,14 +58,15 @@ function checkSubPart(key, uploadId, expectedParts, cb) { function azureCheck(key, cb) { (async () => { - const res = await s3.send(new GetObjectCommand({ - Bucket: azureContainerName, - Key: key, - })); + const res = await s3.send( + new GetObjectCommand({ + Bucket: azureContainerName, + Key: key, + }), + ); assert(res, 'expected getObject response'); assert.strictEqual(res.ETag, `"${expectedMD5}"`); - const properties = await azureClient.getContainerClient(azureContainerName) - .getProperties(key); + const properties = await azureClient.getContainerClient(azureContainerName).getProperties(key); const convertedMD5 = convertMD5(properties.contentSettings.contentMD5); assert.strictEqual(convertedMD5, expectedMD5); })() @@ -63,8 +74,7 @@ function azureCheck(key, cb) { .catch(err => cb(err)); } -describeSkipIfNotMultiple('MultipleBackend put part to AZURE', function -describeF() { +describeSkipIfNotMultiple('MultipleBackend put part to AZURE', function describeF() { this.timeout(80000); withV4(sigCfg => { beforeEach(function beforeFn() { @@ -74,51 +84,65 @@ describeF() { }); describe('with bucket location header', () => { beforeEach(function beforeEachFn(done) { - async.waterfall([ - next => { - s3.send(new CreateBucketCommand({ - Bucket: azureContainerName, - })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new CreateMultipartUploadCommand({ - Bucket: azureContainerName, - Key: this.currentTest.key, - Metadata: { 'scal-location-constraint': azureLocation }, - })) - .then(res => { - this.currentTest.uploadId = res.UploadId; - next(); - }) - .catch(next); - }, - ], done); + async.waterfall( + [ + next => { + s3.send( + new CreateBucketCommand({ + Bucket: azureContainerName, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + s3.send( + new CreateMultipartUploadCommand({ + Bucket: azureContainerName, + Key: this.currentTest.key, + Metadata: { 'scal-location-constraint': azureLocation }, + }), + ) + .then(res => { + this.currentTest.uploadId = res.UploadId; + next(); + }) + .catch(next); + }, + ], + done, + ); }); afterEach(function afterEachFn(done) { - async.waterfall([ - next => { - s3.send(new AbortMultipartUploadCommand({ - Bucket: azureContainerName, - Key: this.currentTest.key, - UploadId: this.currentTest.uploadId, - })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new DeleteBucketCommand({ - Bucket: azureContainerName, - })) - .then(() => next()) - .catch(next); + async.waterfall( + [ + next => { + s3.send( + new AbortMultipartUploadCommand({ + Bucket: azureContainerName, + Key: this.currentTest.key, + UploadId: this.currentTest.uploadId, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + s3.send( + new DeleteBucketCommand({ + Bucket: azureContainerName, + }), + ) + .then(() => next()) + .catch(next); + }, + ], + err => { + assert.equal(err, null, `Error aborting MPU: ${err}`); + done(); }, - ], err => { - assert.equal(err, null, `Error aborting MPU: ${err}`); - done(); - }); + ); }); it('should put 0-byte block to Azure', function itFn(done) { @@ -128,35 +152,44 @@ describeF() { UploadId: this.test.uploadId, PartNumber: 1, }; - async.waterfall([ - next => { - const uploadParams = { - ...params, - Body: Buffer.alloc(0), - }; - s3.send(new UploadPartCommand(uploadParams)) - .then(res => { - const eTagExpected = `"${azureMpuUtils.zeroByteETag}"`; - assert.strictEqual(res.ETag, eTagExpected); - next(); - }) - .catch(next); - }, - next => azureClient.getContainerClient(azureContainerName) - .getBlockBlobClient(this.test.key) - .getBlockList('all').then( - () => assert.fail('Expected failure but got success'), err => { - assert.strictEqual(err.code, 'BlobNotFound'); - next(); - }), - ], done); + async.waterfall( + [ + next => { + const uploadParams = { + ...params, + Body: Buffer.alloc(0), + }; + s3.send(new UploadPartCommand(uploadParams)) + .then(res => { + const eTagExpected = `"${azureMpuUtils.zeroByteETag}"`; + assert.strictEqual(res.ETag, eTagExpected); + next(); + }) + .catch(next); + }, + next => + azureClient + .getContainerClient(azureContainerName) + .getBlockBlobClient(this.test.key) + .getBlockList('all') + .then( + () => assert.fail('Expected failure but got success'), + err => { + assert.strictEqual(err.code, 'BlobNotFound'); + next(); + }, + ), + ], + done, + ); }); it('should put 2 blocks to Azure', function itFn(done) { const body = Buffer.alloc(maxSubPartSize + 10); - const parts = [{ partnbr: 1, subpartnbr: 0, - size: maxSubPartSize }, - { partnbr: 1, subpartnbr: 1, size: 10 }]; + const parts = [ + { partnbr: 1, subpartnbr: 0, size: maxSubPartSize }, + { partnbr: 1, subpartnbr: 1, size: 10 }, + ]; const params = { Bucket: azureContainerName, Key: this.test.key, @@ -164,218 +197,244 @@ describeF() { PartNumber: 1, Body: body, }; - async.waterfall([ - next => { - s3.send(new UploadPartCommand(params)) - .then(res => { - const eTagExpected = expectedETag(body); - assert.strictEqual(res.ETag, eTagExpected); - next(); - }) - .catch(next); - }, - next => checkSubPart(this.test.key, this.test.uploadId, - parts, next), - ], done); + async.waterfall( + [ + next => { + s3.send(new UploadPartCommand(params)) + .then(res => { + const eTagExpected = expectedETag(body); + assert.strictEqual(res.ETag, eTagExpected); + next(); + }) + .catch(next); + }, + next => checkSubPart(this.test.key, this.test.uploadId, parts, next), + ], + done, + ); }); - it('should put 5 parts bigger than maxSubPartSize to Azure', - function it(done) { + it('should put 5 parts bigger than maxSubPartSize to Azure', function it(done) { const body = Buffer.alloc(maxSubPartSize + 10); let parts = []; for (let i = 1; i < 6; i++) { parts = parts.concat([ - { partnbr: i, subpartnbr: 0, size: maxSubPartSize }, - { partnbr: i, subpartnbr: 1, size: 10 }, + { partnbr: i, subpartnbr: 0, size: maxSubPartSize }, + { partnbr: i, subpartnbr: 1, size: 10 }, ]); } - async.times(5, (n, next) => { - const partNumber = n + 1; - const params = { - Bucket: azureContainerName, - Key: this.test.key, - UploadId: this.test.uploadId, - PartNumber: partNumber, - Body: body, - }; - s3.send(new UploadPartCommand(params)) - .then(res => { - const eTagExpected = expectedETag(body); - assert.strictEqual(res.ETag, eTagExpected); - next(); - }) - .catch(next); - }, err => { - assert.equal(err, null, 'Expected success, ' + - `got error: ${err}`); - checkSubPart(this.test.key, this.test.uploadId, - parts, done); - }); + async.times( + 5, + (n, next) => { + const partNumber = n + 1; + const params = { + Bucket: azureContainerName, + Key: this.test.key, + UploadId: this.test.uploadId, + PartNumber: partNumber, + Body: body, + }; + s3.send(new UploadPartCommand(params)) + .then(res => { + const eTagExpected = expectedETag(body); + assert.strictEqual(res.ETag, eTagExpected); + next(); + }) + .catch(next); + }, + err => { + assert.equal(err, null, 'Expected success, ' + `got error: ${err}`); + checkSubPart(this.test.key, this.test.uploadId, parts, done); + }, + ); }); - it('should put 5 parts smaller than maxSubPartSize to Azure', - function it(done) { + it('should put 5 parts smaller than maxSubPartSize to Azure', function it(done) { const body = Buffer.alloc(10); let parts = []; for (let i = 1; i < 6; i++) { - parts = parts.concat([ - { partnbr: i, subpartnbr: 0, size: 10 }, - ]); + parts = parts.concat([{ partnbr: i, subpartnbr: 0, size: 10 }]); } - async.times(5, (n, next) => { - const partNumber = n + 1; - const params = { - Bucket: azureContainerName, - Key: this.test.key, - UploadId: this.test.uploadId, - PartNumber: partNumber, - Body: body, - }; - s3.send(new UploadPartCommand(params)) - .then(res => { - const eTagExpected = expectedETag(body); - assert.strictEqual(res.ETag, eTagExpected); - next(); - }) - .catch(next); - }, err => { - assert.equal(err, null, 'Expected success, ' + - `got error: ${err}`); - checkSubPart(this.test.key, this.test.uploadId, - parts, done); - }); - }); - - it('should put the same part twice', function itFn(done) { - const body1 = Buffer.alloc(maxSubPartSize + 10); - const body2 = Buffer.alloc(20); - const parts2 = [{ partnbr: 1, subpartnbr: 0, size: 20 }, - { partnbr: 1, subpartnbr: 1, size: 10 }]; - async.waterfall([ - next => { - s3.send(new UploadPartCommand({ - Bucket: azureContainerName, - Key: this.test.key, - UploadId: this.test.uploadId, - PartNumber: 1, - Body: body1, - })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new UploadPartCommand({ + async.times( + 5, + (n, next) => { + const partNumber = n + 1; + const params = { Bucket: azureContainerName, Key: this.test.key, UploadId: this.test.uploadId, - PartNumber: 1, - Body: body2, - })) + PartNumber: partNumber, + Body: body, + }; + s3.send(new UploadPartCommand(params)) .then(res => { - const eTagExpected = expectedETag(body2); + const eTagExpected = expectedETag(body); assert.strictEqual(res.ETag, eTagExpected); next(); }) .catch(next); }, - next => checkSubPart(this.test.key, this.test.uploadId, - parts2, next), - ], done); + err => { + assert.equal(err, null, 'Expected success, ' + `got error: ${err}`); + checkSubPart(this.test.key, this.test.uploadId, parts, done); + }, + ); + }); + + it('should put the same part twice', function itFn(done) { + const body1 = Buffer.alloc(maxSubPartSize + 10); + const body2 = Buffer.alloc(20); + const parts2 = [ + { partnbr: 1, subpartnbr: 0, size: 20 }, + { partnbr: 1, subpartnbr: 1, size: 10 }, + ]; + async.waterfall( + [ + next => { + s3.send( + new UploadPartCommand({ + Bucket: azureContainerName, + Key: this.test.key, + UploadId: this.test.uploadId, + PartNumber: 1, + Body: body1, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + s3.send( + new UploadPartCommand({ + Bucket: azureContainerName, + Key: this.test.key, + UploadId: this.test.uploadId, + PartNumber: 1, + Body: body2, + }), + ) + .then(res => { + const eTagExpected = expectedETag(body2); + assert.strictEqual(res.ETag, eTagExpected); + next(); + }) + .catch(next); + }, + next => checkSubPart(this.test.key, this.test.uploadId, parts2, next), + ], + done, + ); }); }); describe('with same key as preexisting part', () => { beforeEach(function beforeEachFn(done) { - async.waterfall([ - next => { - s3.send(new CreateBucketCommand({ - Bucket: azureContainerName, - })) - .then(() => next()) - .catch(next); - }, - next => { - const body = Buffer.alloc(10); - s3.send(new PutObjectCommand({ - Bucket: azureContainerName, - Key: this.currentTest.key, - Metadata: { 'scal-location-constraint': - azureLocation }, - Body: body, - })) - .then(() => next()) - .catch(err => { - assert.equal(err, null, 'Err putting object to ' + - `azure: ${err}`); - next(err); - }); - }, - next => { - s3.send(new CreateMultipartUploadCommand({ - Bucket: azureContainerName, - Key: this.currentTest.key, - Metadata: { 'scal-location-constraint': azureLocation }, - })) - .then(res => { - this.currentTest.uploadId = res.UploadId; - next(); - }) - .catch(next); - }, - ], done); + async.waterfall( + [ + next => { + s3.send( + new CreateBucketCommand({ + Bucket: azureContainerName, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + const body = Buffer.alloc(10); + s3.send( + new PutObjectCommand({ + Bucket: azureContainerName, + Key: this.currentTest.key, + Metadata: { 'scal-location-constraint': azureLocation }, + Body: body, + }), + ) + .then(() => next()) + .catch(err => { + assert.equal(err, null, 'Err putting object to ' + `azure: ${err}`); + next(err); + }); + }, + next => { + s3.send( + new CreateMultipartUploadCommand({ + Bucket: azureContainerName, + Key: this.currentTest.key, + Metadata: { 'scal-location-constraint': azureLocation }, + }), + ) + .then(res => { + this.currentTest.uploadId = res.UploadId; + next(); + }) + .catch(next); + }, + ], + done, + ); }); afterEach(function afterEachFn(done) { - async.waterfall([ - next => { - process.stdout.write('Aborting multipart upload\n'); - s3.send(new AbortMultipartUploadCommand({ - Bucket: azureContainerName, - Key: this.currentTest.key, - UploadId: this.currentTest.uploadId, - })) - .then(() => next()) - .catch(next); - }, - next => { - process.stdout.write('Deleting object\n'); - s3.send(new DeleteObjectCommand({ - Bucket: azureContainerName, - Key: this.currentTest.key, - })) - .then(() => next()) - .catch(next); - }, - next => { - process.stdout.write('Deleting bucket\n'); - s3.send(new DeleteBucketCommand({ - Bucket: azureContainerName, - })) - .then(() => next()) - .catch(next); + async.waterfall( + [ + next => { + process.stdout.write('Aborting multipart upload\n'); + s3.send( + new AbortMultipartUploadCommand({ + Bucket: azureContainerName, + Key: this.currentTest.key, + UploadId: this.currentTest.uploadId, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + process.stdout.write('Deleting object\n'); + s3.send( + new DeleteObjectCommand({ + Bucket: azureContainerName, + Key: this.currentTest.key, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + process.stdout.write('Deleting bucket\n'); + s3.send( + new DeleteBucketCommand({ + Bucket: azureContainerName, + }), + ) + .then(() => next()) + .catch(next); + }, + ], + err => { + assert.equal(err, null, `Err in afterEach: ${err}`); + done(); }, - ], err => { - assert.equal(err, null, `Err in afterEach: ${err}`); - done(); - }); + ); }); - it('should put a part without overwriting existing object', - function itFn(done) { + it('should put a part without overwriting existing object', function itFn(done) { const body = Buffer.alloc(20); - s3.send(new UploadPartCommand({ - Bucket: azureContainerName, - Key: this.test.key, - UploadId: this.test.uploadId, - PartNumber: 1, - Body: body, - })) + s3.send( + new UploadPartCommand({ + Bucket: azureContainerName, + Key: this.test.key, + UploadId: this.test.uploadId, + PartNumber: 1, + Body: body, + }), + ) .then(() => { azureCheck(this.test.key, done); }) .catch(err => { - assert.strictEqual(err, null, 'Err putting part to ' + - `Azure: ${err}`); + assert.strictEqual(err, null, 'Err putting part to ' + `Azure: ${err}`); done(err); }); }); @@ -383,94 +442,107 @@ describeF() { }); }); -describeSkipIfNotMultiple('MultipleBackend put part to AZURE ' + -'location with bucketMatch sets to false', function -describeF() { - this.timeout(80000); - withV4(sigCfg => { - beforeEach(function beforeFn() { - this.currentTest.key = uniqName(keyObject); - bucketUtil = new BucketUtility('default', sigCfg); - s3 = bucketUtil.s3; - }); - describe('with bucket location header', () => { - beforeEach(function beforeEachFn(done) { - async.waterfall([ - next => { - s3.send(new CreateBucketCommand({ - Bucket: azureContainerName, - })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new CreateMultipartUploadCommand({ - Bucket: azureContainerName, - Key: this.currentTest.key, - Metadata: { 'scal-location-constraint': - azureLocationMismatch }, - })) - .then(res => { - this.currentTest.uploadId = res.UploadId; - next(); - }) - .catch(next); - }, - ], done); +describeSkipIfNotMultiple( + 'MultipleBackend put part to AZURE ' + 'location with bucketMatch sets to false', + function describeF() { + this.timeout(80000); + withV4(sigCfg => { + beforeEach(function beforeFn() { + this.currentTest.key = uniqName(keyObject); + bucketUtil = new BucketUtility('default', sigCfg); + s3 = bucketUtil.s3; }); + describe('with bucket location header', () => { + beforeEach(function beforeEachFn(done) { + async.waterfall( + [ + next => { + s3.send( + new CreateBucketCommand({ + Bucket: azureContainerName, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + s3.send( + new CreateMultipartUploadCommand({ + Bucket: azureContainerName, + Key: this.currentTest.key, + Metadata: { 'scal-location-constraint': azureLocationMismatch }, + }), + ) + .then(res => { + this.currentTest.uploadId = res.UploadId; + next(); + }) + .catch(next); + }, + ], + done, + ); + }); - afterEach(function afterEachFn(done) { - async.waterfall([ - next => { - s3.send(new AbortMultipartUploadCommand({ - Bucket: azureContainerName, - Key: this.currentTest.key, - UploadId: this.currentTest.uploadId, - })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new DeleteBucketCommand({ - Bucket: azureContainerName, - })) - .then(() => next()) - .catch(next); - }, - ], err => { - assert.equal(err, null, `Error aborting MPU: ${err}`); - done(); + afterEach(function afterEachFn(done) { + async.waterfall( + [ + next => { + s3.send( + new AbortMultipartUploadCommand({ + Bucket: azureContainerName, + Key: this.currentTest.key, + UploadId: this.currentTest.uploadId, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + s3.send( + new DeleteBucketCommand({ + Bucket: azureContainerName, + }), + ) + .then(() => next()) + .catch(next); + }, + ], + err => { + assert.equal(err, null, `Error aborting MPU: ${err}`); + done(); + }, + ); }); - }); - it('should put block to AZURE location with bucketMatch' + - ' sets to false', function itFn(done) { - const body20 = Buffer.alloc(20); - const params = { - Bucket: azureContainerName, - Key: this.test.key, - UploadId: this.test.uploadId, - PartNumber: 1, - Body: body20, - }; - const parts = [{ partnbr: 1, subpartnbr: 0, - size: 20 }]; - async.waterfall([ - next => { - s3.send(new UploadPartCommand(params)) - .then(res => { - const eTagExpected = - '"441018525208457705bf09a8ee3c1093"'; - assert.strictEqual(res.ETag, eTagExpected); - next(); - }) - .catch(next); - }, - next => checkSubPart( - `${azureContainerName}/${this.test.key}`, - this.test.uploadId, parts, next), - ], done); + it('should put block to AZURE location with bucketMatch' + ' sets to false', function itFn(done) { + const body20 = Buffer.alloc(20); + const params = { + Bucket: azureContainerName, + Key: this.test.key, + UploadId: this.test.uploadId, + PartNumber: 1, + Body: body20, + }; + const parts = [{ partnbr: 1, subpartnbr: 0, size: 20 }]; + async.waterfall( + [ + next => { + s3.send(new UploadPartCommand(params)) + .then(res => { + const eTagExpected = '"441018525208457705bf09a8ee3c1093"'; + assert.strictEqual(res.ETag, eTagExpected); + next(); + }) + .catch(next); + }, + next => + checkSubPart(`${azureContainerName}/${this.test.key}`, this.test.uploadId, parts, next), + ], + done, + ); + }); }); }); - }); -}); + }, +); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/mpuParts/putPartGcp.js b/tests/functional/aws-node-sdk/test/multipleBackend/mpuParts/putPartGcp.js index cbd8222074..03c8d411a7 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/mpuParts/putPartGcp.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/mpuParts/putPartGcp.js @@ -13,10 +13,16 @@ const arsenal = require('arsenal'); const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); -const { gcpClient, gcpBucket, gcpBucketMPU, - gcpLocation, gcpLocationMismatch, uniqName, genUniqID, - describeSkipIfNotMultiple } - = require('../utils'); +const { + gcpClient, + gcpBucket, + gcpBucketMPU, + gcpLocation, + gcpLocationMismatch, + uniqName, + genUniqID, + describeSkipIfNotMultiple, +} = require('../utils'); const { createMpuKey } = arsenal.storage.data.external.GcpUtils; const keyObject = 'putgcp'; @@ -35,20 +41,16 @@ function checkMPUResult(bucket, key, uploadId, objCount, expected, cb) { UploadId: uploadId, }; gcpClient.listParts(params, (err, res) => { - assert.ifError(err, - `Expected success, but got err ${err}`); - assert((res && res.Contents && - res.Contents.length === objCount)); + assert.ifError(err, `Expected success, but got err ${err}`); + assert(res && res.Contents && res.Contents.length === objCount); res.Contents.forEach(part => { - assert.strictEqual( - part.ETag, `"${expected}"`); + assert.strictEqual(part.ETag, `"${expected}"`); }); cb(); }); } -describeSkipIfNotMultiple('MultipleBacked put part to GCP', function -describeFn() { +describeSkipIfNotMultiple('MultipleBacked put part to GCP', function describeFn() { this.timeout(180000); withV4(sigCfg => { beforeEach(function beforeFn() { @@ -59,47 +61,57 @@ describeFn() { describe('with bucket location header', () => { beforeEach(function beforeEachFn(done) { - async.waterfall([ - next => { - s3.send(new CreateBucketCommand({ Bucket: bucket })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: this.currentTest.key, - Metadata: { 'scal-location-constraint': gcpLocation }, - })) - .then(res => { - this.currentTest.uploadId = res.UploadId; - next(); - }) - .catch(next); - }, - ], done); + async.waterfall( + [ + next => { + s3.send(new CreateBucketCommand({ Bucket: bucket })) + .then(() => next()) + .catch(next); + }, + next => { + s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: this.currentTest.key, + Metadata: { 'scal-location-constraint': gcpLocation }, + }), + ) + .then(res => { + this.currentTest.uploadId = res.UploadId; + next(); + }) + .catch(next); + }, + ], + done, + ); }); afterEach(function afterEachFn(done) { - async.waterfall([ - next => { - s3.send(new AbortMultipartUploadCommand({ - Bucket: bucket, - Key: this.currentTest.key, - UploadId: this.currentTest.uploadId, - })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new DeleteBucketCommand({ Bucket: bucket })) - .then(() => next()) - .catch(next); + async.waterfall( + [ + next => { + s3.send( + new AbortMultipartUploadCommand({ + Bucket: bucket, + Key: this.currentTest.key, + UploadId: this.currentTest.uploadId, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + s3.send(new DeleteBucketCommand({ Bucket: bucket })) + .then(() => next()) + .catch(next); + }, + ], + err => { + assert.equal(err, null, `Error aborting MPU: ${err}`); + done(); }, - ], err => { - assert.equal(err, null, `Error aborting MPU: ${err}`); - done(); - }); + ); }); it('should put 0-byte part to GCP', function itFn(done) { @@ -109,282 +121,320 @@ describeFn() { UploadId: this.test.uploadId, PartNumber: 1, }; - async.waterfall([ - next => { - s3.send(new UploadPartCommand({ - ...params, - Body: Buffer.alloc(0), - })) - .then(res => { + async.waterfall( + [ + next => { + s3.send( + new UploadPartCommand({ + ...params, + Body: Buffer.alloc(0), + }), + ) + .then(res => { + assert.strictEqual(res.ETag, `"${emptyMD5}"`); + next(); + }) + .catch(next); + }, + next => { + const mpuKey = createMpuKey(this.test.key, this.test.uploadId, 1); + const getParams = { + Bucket: gcpBucketMPU, + Key: mpuKey, + }; + gcpClient.getObject(getParams, (err, res) => { + assert.ifError(err, `Expected success, but got err ${err}`); assert.strictEqual(res.ETag, `"${emptyMD5}"`); next(); - }) - .catch(next); - }, - next => { - const mpuKey = - createMpuKey(this.test.key, this.test.uploadId, 1); - const getParams = { - Bucket: gcpBucketMPU, - Key: mpuKey, - }; - gcpClient.getObject(getParams, (err, res) => { - assert.ifError(err, - `Expected success, but got err ${err}`); - assert.strictEqual(res.ETag, `"${emptyMD5}"`); - next(); - }); - }, - ], done); + }); + }, + ], + done, + ); }); it('should put 2 parts to GCP', function ifFn(done) { - async.waterfall([ - next => { - async.times(2, (n, cb) => { - const params = { - Bucket: bucket, - Key: this.test.key, - UploadId: this.test.uploadId, - Body: body, - PartNumber: n + 1, - }; - s3.send(new UploadPartCommand(params)) - .then(res => { - assert.strictEqual( - res.ETag, `"${correctMD5}"`); - cb(); - }) - .catch(cb); - }, err => next(err)); - }, - next => checkMPUResult( - gcpBucketMPU, this.test.key, this.test.uploadId, - 2, correctMD5, next), - ], done); + async.waterfall( + [ + next => { + async.times( + 2, + (n, cb) => { + const params = { + Bucket: bucket, + Key: this.test.key, + UploadId: this.test.uploadId, + Body: body, + PartNumber: n + 1, + }; + s3.send(new UploadPartCommand(params)) + .then(res => { + assert.strictEqual(res.ETag, `"${correctMD5}"`); + cb(); + }) + .catch(cb); + }, + err => next(err), + ); + }, + next => checkMPUResult(gcpBucketMPU, this.test.key, this.test.uploadId, 2, correctMD5, next), + ], + done, + ); }); it('should put the same part twice', function ifFn(done) { - async.waterfall([ - next => { - const partBody = ['', body]; - const partMD5 = [emptyMD5, correctMD5]; - async.timesSeries(2, (n, cb) => { - const params = { - Bucket: bucket, - Key: this.test.key, - UploadId: this.test.uploadId, - Body: partBody[n], - PartNumber: 1, - }; - s3.send(new UploadPartCommand(params)) - .then(res => { - assert.strictEqual( - res.ETag, `"${partMD5[n]}"`); - cb(); - }) - .catch(cb); - }, err => next(err)); - }, - next => checkMPUResult( - gcpBucketMPU, this.test.key, this.test.uploadId, - 1, correctMD5, next), - ], done); + async.waterfall( + [ + next => { + const partBody = ['', body]; + const partMD5 = [emptyMD5, correctMD5]; + async.timesSeries( + 2, + (n, cb) => { + const params = { + Bucket: bucket, + Key: this.test.key, + UploadId: this.test.uploadId, + Body: partBody[n], + PartNumber: 1, + }; + s3.send(new UploadPartCommand(params)) + .then(res => { + assert.strictEqual(res.ETag, `"${partMD5[n]}"`); + cb(); + }) + .catch(cb); + }, + err => next(err), + ); + }, + next => checkMPUResult(gcpBucketMPU, this.test.key, this.test.uploadId, 1, correctMD5, next), + ], + done, + ); }); }); describe('with same key as preexisting part', () => { beforeEach(function beforeEachFn(done) { - async.waterfall([ - next => { - s3.send(new CreateBucketCommand({ Bucket: bucket })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: this.currentTest.key, - Metadata: { - 'scal-location-constraint': gcpLocation }, - Body: body, - })) - .then(() => next()) - .catch(err => next(new Error( - `Err putting object to GCP: ${err}`))); - }, - next => { - s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: this.currentTest.key, - Metadata: { 'scal-location-constraint': gcpLocation }, - })) - .then(res => { - this.currentTest.uploadId = res.UploadId; - next(); - }) - .catch(next); - }, - ], done); + async.waterfall( + [ + next => { + s3.send(new CreateBucketCommand({ Bucket: bucket })) + .then(() => next()) + .catch(next); + }, + next => { + s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: this.currentTest.key, + Metadata: { + 'scal-location-constraint': gcpLocation, + }, + Body: body, + }), + ) + .then(() => next()) + .catch(err => next(new Error(`Err putting object to GCP: ${err}`))); + }, + next => { + s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: this.currentTest.key, + Metadata: { 'scal-location-constraint': gcpLocation }, + }), + ) + .then(res => { + this.currentTest.uploadId = res.UploadId; + next(); + }) + .catch(next); + }, + ], + done, + ); }); afterEach(function afterEachFn(done) { - async.waterfall([ - next => { - process.stdout.write('Aborting multipart upload\n'); - s3.send(new AbortMultipartUploadCommand({ - Bucket: bucket, - Key: this.currentTest.key, - UploadId: this.currentTest.uploadId, - })) - .then(() => next()) - .catch(next); - }, - next => { - process.stdout.write('Deleting object\n'); - s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: this.currentTest.key, - })) - .then(() => next()) - .catch(next); + async.waterfall( + [ + next => { + process.stdout.write('Aborting multipart upload\n'); + s3.send( + new AbortMultipartUploadCommand({ + Bucket: bucket, + Key: this.currentTest.key, + UploadId: this.currentTest.uploadId, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + process.stdout.write('Deleting object\n'); + s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: this.currentTest.key, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + process.stdout.write('Deleting bucket\n'); + s3.send( + new DeleteBucketCommand({ + Bucket: bucket, + }), + ) + .then(() => next()) + .catch(next); + }, + ], + err => { + assert.equal(err, null, `Err in afterEach: ${err}`); + done(); }, - next => { - process.stdout.write('Deleting bucket\n'); - s3.send(new DeleteBucketCommand({ - Bucket: bucket, - })) - .then(() => next()) - .catch(next); - }, - ], err => { - assert.equal(err, null, `Err in afterEach: ${err}`); - done(); - }); + ); }); - it('should put a part without overwriting existing object', - function itFn(done) { + it('should put a part without overwriting existing object', function itFn(done) { const body = Buffer.alloc(20); - s3.send(new UploadPartCommand({ - Bucket: bucket, - Key: this.test.key, - UploadId: this.test.uploadId, - PartNumber: 1, - Body: body, - })) + s3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: this.test.key, + UploadId: this.test.uploadId, + PartNumber: 1, + Body: body, + }), + ) .then(() => { - gcpClient.getObject({ - Bucket: gcpBucket, - Key: this.test.key, - }, (err, res) => { - assert.ifError(err, - `Expected success, but got err ${err}`); - assert.strictEqual(res.ETag, `"${correctMD5}"`); - done(); - }); + gcpClient.getObject( + { + Bucket: gcpBucket, + Key: this.test.key, + }, + (err, res) => { + assert.ifError(err, `Expected success, but got err ${err}`); + assert.strictEqual(res.ETag, `"${correctMD5}"`); + done(); + }, + ); }) - .catch(err => done(new Error( - `Err putting part to GCP: ${err}`))); + .catch(err => done(new Error(`Err putting part to GCP: ${err}`))); }); }); }); }); -describeSkipIfNotMultiple('MultipleBackend put part to GCP location ' + -'with bucketMatch sets to false', function -describeF() { - this.timeout(80000); - withV4(sigCfg => { - beforeEach(function beforeFn() { - this.currentTest.key = uniqName(keyObject); - bucketUtil = new BucketUtility('default', sigCfg); - s3 = bucketUtil.s3; - }); - describe('with bucket location header', () => { - beforeEach(function beforeEachFn(done) { - async.waterfall([ - next => { - s3.send(new CreateBucketCommand({ Bucket: bucket })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: this.currentTest.key, - Metadata: { 'scal-location-constraint': - gcpLocationMismatch }, - })) - .then(res => { - this.currentTest.uploadId = res.UploadId; - next(); - }) - .catch(next); - }, - ], done); +describeSkipIfNotMultiple( + 'MultipleBackend put part to GCP location ' + 'with bucketMatch sets to false', + function describeF() { + this.timeout(80000); + withV4(sigCfg => { + beforeEach(function beforeFn() { + this.currentTest.key = uniqName(keyObject); + bucketUtil = new BucketUtility('default', sigCfg); + s3 = bucketUtil.s3; }); + describe('with bucket location header', () => { + beforeEach(function beforeEachFn(done) { + async.waterfall( + [ + next => { + s3.send(new CreateBucketCommand({ Bucket: bucket })) + .then(() => next()) + .catch(next); + }, + next => { + s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: this.currentTest.key, + Metadata: { 'scal-location-constraint': gcpLocationMismatch }, + }), + ) + .then(res => { + this.currentTest.uploadId = res.UploadId; + next(); + }) + .catch(next); + }, + ], + done, + ); + }); - afterEach(function afterEachFn(done) { - async.waterfall([ - next => { - s3.send(new AbortMultipartUploadCommand({ - Bucket: bucket, - Key: this.currentTest.key, - UploadId: this.currentTest.uploadId, - })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new DeleteBucketCommand({ Bucket: bucket })) - .then(() => next()) - .catch(next); - }, - ], err => { - assert.equal(err, null, `Error aborting MPU: ${err}`); - done(); + afterEach(function afterEachFn(done) { + async.waterfall( + [ + next => { + s3.send( + new AbortMultipartUploadCommand({ + Bucket: bucket, + Key: this.currentTest.key, + UploadId: this.currentTest.uploadId, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + s3.send(new DeleteBucketCommand({ Bucket: bucket })) + .then(() => next()) + .catch(next); + }, + ], + err => { + assert.equal(err, null, `Error aborting MPU: ${err}`); + done(); + }, + ); }); - }); - it('should put part to GCP location with bucketMatch' + - ' sets to false', function itFn(done) { - const body20 = Buffer.alloc(20); - const params = { - Bucket: bucket, - Key: this.test.key, - UploadId: this.test.uploadId, - PartNumber: 1, - Body: body20, - }; - const eTagExpected = - '"441018525208457705bf09a8ee3c1093"'; - async.waterfall([ - next => { - s3.send(new UploadPartCommand(params)) - .then(res => { - assert.strictEqual(res.ETag, eTagExpected); - next(); - }) - .catch(next); - }, - next => { - const key = - createMpuKey(this.test.key, this.test.uploadId, 1); - const mpuKey = `${bucket}/${key}`; - const getParams = { - Bucket: gcpBucketMPU, - Key: mpuKey, - }; - gcpClient.getObject(getParams, (err, res) => { - assert.ifError(err, - `Expected success, but got err ${err}`); - assert.strictEqual(res.ETag, eTagExpected); - next(); - }); - }, - ], done); + it('should put part to GCP location with bucketMatch' + ' sets to false', function itFn(done) { + const body20 = Buffer.alloc(20); + const params = { + Bucket: bucket, + Key: this.test.key, + UploadId: this.test.uploadId, + PartNumber: 1, + Body: body20, + }; + const eTagExpected = '"441018525208457705bf09a8ee3c1093"'; + async.waterfall( + [ + next => { + s3.send(new UploadPartCommand(params)) + .then(res => { + assert.strictEqual(res.ETag, eTagExpected); + next(); + }) + .catch(next); + }, + next => { + const key = createMpuKey(this.test.key, this.test.uploadId, 1); + const mpuKey = `${bucket}/${key}`; + const getParams = { + Bucket: gcpBucketMPU, + Key: mpuKey, + }; + gcpClient.getObject(getParams, (err, res) => { + assert.ifError(err, `Expected success, but got err ${err}`); + assert.strictEqual(res.ETag, eTagExpected); + next(); + }); + }, + ], + done, + ); + }); }); }); - }); -}); + }, +); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/objectCopy/azureObjectCopy.js b/tests/functional/aws-node-sdk/test/multipleBackend/objectCopy/azureObjectCopy.js index a2fdfe3cec..2d37353d23 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/objectCopy/azureObjectCopy.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/objectCopy/azureObjectCopy.js @@ -1,11 +1,6 @@ const assert = require('assert'); const async = require('async'); -const { - CreateBucketCommand, - PutObjectCommand, - GetObjectCommand, - CopyObjectCommand, -} = require('@aws-sdk/client-s3'); +const { CreateBucketCommand, PutObjectCommand, GetObjectCommand, CopyObjectCommand } = require('@aws-sdk/client-s3'); const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); @@ -22,8 +17,7 @@ const { genUniqID, describeSkipIfNotMultiple, } = require('../utils'); -const { createEncryptedBucketPromise } = - require('../../../lib/utility/createEncryptedBucket'); +const { createEncryptedBucketPromise } = require('../../../lib/utility/createEncryptedBucket'); const azureClient = getAzureClient(); const azureContainerName = getAzureContainerName(azureLocation); @@ -60,7 +54,9 @@ function normalizeMetadata(metadata) { } function putSourceObj(key, location, objSize, bucket, cb) { - const sourceParams = { Bucket: bucket, Key: key, + const sourceParams = { + Bucket: bucket, + Key: key, Metadata: { 'test-header': 'copyme', }, @@ -87,69 +83,84 @@ function putSourceObj(key, location, objSize, bucket, cb) { .catch(err => cb(new Error(`Error putting source object: ${err}`))); } -function assertGetObjects(sourceKey, sourceBucket, sourceLoc, destKey, -destBucket, destLoc, azureKey, mdDirective, objSize, callback) { +function assertGetObjects( + sourceKey, + sourceBucket, + sourceLoc, + destKey, + destBucket, + destLoc, + azureKey, + mdDirective, + objSize, + callback, +) { const sourceGetParams = { Bucket: sourceBucket, Key: sourceKey }; const destGetParams = { Bucket: destBucket, Key: destKey }; - async.series([ - cb => { - s3.send(new GetObjectCommand(sourceGetParams)) - .then(res => cb(null, res)) - .catch(cb); - }, - cb => { - s3.send(new GetObjectCommand(destGetParams)) - .then(res => cb(null, res)) - .catch(cb); - }, - cb => azureClient.getContainerClient(azureContainerName) - .getProperties(azureKey) - .then(res => cb(null, res), err => cb(err)), - ], (err, results) => { - assert.equal(err, null, `Error in assertGetObjects: ${err}`); - const [sourceRes, destRes, azureRes] = results; - const sourceMetadata = normalizeMetadata(sourceRes.Metadata); - const destMetadata = normalizeMetadata(destRes.Metadata); - const convertedMD5 = convertMD5(azureRes[0].contentSettings.contentMD5); - if (objSize && objSize.empty) { - assert.strictEqual(sourceRes.ETag, `"${emptyMD5}"`); - assert.strictEqual(destRes.ETag, `"${emptyMD5}"`); - assert.strictEqual(convertedMD5, `${emptyMD5}`); - assert.strictEqual('0', azureRes[0].contentLength); - } else if (objSize && objSize.big) { - assert.strictEqual(sourceRes.ETag, `"${bigMD5}"`); - assert.strictEqual(destRes.ETag, `"${bigMD5}"`); - if (process.env.ENABLE_KMS_ENCRYPTION === 'true') { - assert.strictEqual(sourceRes.ServerSideEncryption, 'AES256'); - assert.strictEqual(destRes.ServerSideEncryption, 'AES256'); + async.series( + [ + cb => { + s3.send(new GetObjectCommand(sourceGetParams)) + .then(res => cb(null, res)) + .catch(cb); + }, + cb => { + s3.send(new GetObjectCommand(destGetParams)) + .then(res => cb(null, res)) + .catch(cb); + }, + cb => + azureClient + .getContainerClient(azureContainerName) + .getProperties(azureKey) + .then( + res => cb(null, res), + err => cb(err), + ), + ], + (err, results) => { + assert.equal(err, null, `Error in assertGetObjects: ${err}`); + const [sourceRes, destRes, azureRes] = results; + const sourceMetadata = normalizeMetadata(sourceRes.Metadata); + const destMetadata = normalizeMetadata(destRes.Metadata); + const convertedMD5 = convertMD5(azureRes[0].contentSettings.contentMD5); + if (objSize && objSize.empty) { + assert.strictEqual(sourceRes.ETag, `"${emptyMD5}"`); + assert.strictEqual(destRes.ETag, `"${emptyMD5}"`); + assert.strictEqual(convertedMD5, `${emptyMD5}`); + assert.strictEqual('0', azureRes[0].contentLength); + } else if (objSize && objSize.big) { + assert.strictEqual(sourceRes.ETag, `"${bigMD5}"`); + assert.strictEqual(destRes.ETag, `"${bigMD5}"`); + if (process.env.ENABLE_KMS_ENCRYPTION === 'true') { + assert.strictEqual(sourceRes.ServerSideEncryption, 'AES256'); + assert.strictEqual(destRes.ServerSideEncryption, 'AES256'); + } else { + assert.strictEqual(convertedMD5, `${bigMD5}`); + } } else { - assert.strictEqual(convertedMD5, `${bigMD5}`); + if (process.env.ENABLE_KMS_ENCRYPTION === 'true') { + assert.strictEqual(sourceRes.ServerSideEncryption, 'AES256'); + assert.strictEqual(destRes.ServerSideEncryption, 'AES256'); + } else { + assert.strictEqual(sourceRes.ETag, `"${normalMD5}"`); + assert.strictEqual(destRes.ETag, `"${normalMD5}"`); + assert.strictEqual(convertedMD5, `${normalMD5}`); + } } - } else { - if (process.env.ENABLE_KMS_ENCRYPTION === 'true') { - assert.strictEqual(sourceRes.ServerSideEncryption, 'AES256'); - assert.strictEqual(destRes.ServerSideEncryption, 'AES256'); - } else { - assert.strictEqual(sourceRes.ETag, `"${normalMD5}"`); - assert.strictEqual(destRes.ETag, `"${normalMD5}"`); - assert.strictEqual(convertedMD5, `${normalMD5}`); + if (mdDirective === 'COPY') { + assert.strictEqual(sourceMetadata['test-header'], destMetadata['test-header']); + assert.strictEqual(azureRes[0].metadata.test_header, destMetadata['test-header']); } - } - if (mdDirective === 'COPY') { - assert.strictEqual(sourceMetadata['test-header'], - destMetadata['test-header']); - assert.strictEqual(azureRes[0].metadata.test_header, - destMetadata['test-header']); - } - assert.strictEqual(sourceRes.ContentLength, destRes.ContentLength); - assert.strictEqual(sourceMetadata[locMetaHeader], sourceLoc); - assert.strictEqual(destMetadata[locMetaHeader], destLoc); - callback(); - }); + assert.strictEqual(sourceRes.ContentLength, destRes.ContentLength); + assert.strictEqual(sourceMetadata[locMetaHeader], sourceLoc); + assert.strictEqual(destMetadata[locMetaHeader], destLoc); + callback(); + }, + ); } -describeSkipIfNotMultiple('MultipleBackend object copy: Azure', -function testSuite() { +describeSkipIfNotMultiple('MultipleBackend object copy: Azure', function testSuite() { this.timeout(250000); withV4(sigCfg => { beforeEach(function beFn() { @@ -158,43 +169,48 @@ function testSuite() { bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; process.stdout.write('Creating bucket\n'); - s3.createBucketPromise = params => - s3.send(new CreateBucketCommand(params)); + s3.createBucketPromise = params => s3.send(new CreateBucketCommand(params)); if (process.env.ENABLE_KMS_ENCRYPTION === 'true') { s3.createBucketPromise = createEncryptedBucketPromise; } - return s3.createBucketPromise({ Bucket: bucket, - CreateBucketConfiguration: { - LocationConstraint: memLocation, - }, - }) - .then(() => s3.createBucketPromise({ Bucket: bucketAzure, - CreateBucketConfiguration: { - LocationConstraint: azureLocation, - }, - })) - .catch(err => { - process.stdout.write(`Error creating bucket: ${err}\n`); - throw err; - }); + return s3 + .createBucketPromise({ + Bucket: bucket, + CreateBucketConfiguration: { + LocationConstraint: memLocation, + }, + }) + .then(() => + s3.createBucketPromise({ + Bucket: bucketAzure, + CreateBucketConfiguration: { + LocationConstraint: azureLocation, + }, + }), + ) + .catch(err => { + process.stdout.write(`Error creating bucket: ${err}\n`); + throw err; + }); }); afterEach(() => { process.stdout.write('Emptying bucket\n'); - return bucketUtil.empty(bucket) - .then(() => bucketUtil.empty(bucketAzure)) - .then(() => { - process.stdout.write(`Deleting bucket: ${bucket}\n`); - return bucketUtil.deleteOne(bucket); - }) - .then(() => { - process.stdout.write(`Deleting bucket: ${bucketAzure}\n`); - return bucketUtil.deleteOne(bucketAzure); - }) - .catch(err => { - process.stdout.write(`Error in afterEach: ${err}\n`); - throw err; - }); + return bucketUtil + .empty(bucket) + .then(() => bucketUtil.empty(bucketAzure)) + .then(() => { + process.stdout.write(`Deleting bucket: ${bucket}\n`); + return bucketUtil.deleteOne(bucket); + }) + .then(() => { + process.stdout.write(`Deleting bucket: ${bucketAzure}\n`); + return bucketUtil.deleteOne(bucketAzure); + }) + .catch(err => { + process.stdout.write(`Error in afterEach: ${err}\n`); + throw err; + }); }); it('should copy an object from mem to Azure', function itFn(done) { @@ -208,19 +224,25 @@ function testSuite() { }; s3.send(new CopyObjectCommand(copyParams)) .then(result => { - assert.strictEqual(result.CopyObjectResult.ETag, - `"${normalMD5}"`); - assertGetObjects(this.test.key, bucket, memLocation, - this.test.copyKey, bucket, azureLocation, - this.test.copyKey, 'REPLACE', null, done); + assert.strictEqual(result.CopyObjectResult.ETag, `"${normalMD5}"`); + assertGetObjects( + this.test.key, + bucket, + memLocation, + this.test.copyKey, + bucket, + azureLocation, + this.test.copyKey, + 'REPLACE', + null, + done, + ); }) - .catch(err => done(new Error( - `Expected success but got error: ${err}`))); + .catch(err => done(new Error(`Expected success but got error: ${err}`))); }); }); - it('should copy an object with no location contraint from mem to Azure', - function itFn(done) { + it('should copy an object with no location contraint from mem to Azure', function itFn(done) { putSourceObj(this.test.key, null, null, bucket, () => { const copyParams = { Bucket: bucketAzure, @@ -230,14 +252,21 @@ function testSuite() { }; s3.send(new CopyObjectCommand(copyParams)) .then(result => { - assert.strictEqual(result.CopyObjectResult.ETag, - `"${normalMD5}"`); - assertGetObjects(this.test.key, bucket, undefined, - this.test.copyKey, bucketAzure, undefined, - this.test.copyKey, 'COPY', null, done); + assert.strictEqual(result.CopyObjectResult.ETag, `"${normalMD5}"`); + assertGetObjects( + this.test.key, + bucket, + undefined, + this.test.copyKey, + bucketAzure, + undefined, + this.test.copyKey, + 'COPY', + null, + done, + ); }) - .catch(err => done(new Error( - `Expected success but got error: ${err}`))); + .catch(err => done(new Error(`Expected success but got error: ${err}`))); }); }); @@ -252,14 +281,21 @@ function testSuite() { }; s3.send(new CopyObjectCommand(copyParams)) .then(result => { - assert.strictEqual(result.CopyObjectResult.ETag, - `"${normalMD5}"`); - assertGetObjects(this.test.key, bucket, azureLocation, - this.test.copyKey, bucket, memLocation, - this.test.key, 'REPLACE', null, done); + assert.strictEqual(result.CopyObjectResult.ETag, `"${normalMD5}"`); + assertGetObjects( + this.test.key, + bucket, + azureLocation, + this.test.copyKey, + bucket, + memLocation, + this.test.key, + 'REPLACE', + null, + done, + ); }) - .catch(err => done(new Error( - `Expected success but got error: ${err}`))); + .catch(err => done(new Error(`Expected success but got error: ${err}`))); }); }); @@ -274,14 +310,21 @@ function testSuite() { }; s3.send(new CopyObjectCommand(copyParams)) .then(result => { - assert.strictEqual(result.CopyObjectResult.ETag, - `"${normalMD5}"`); - assertGetObjects(this.test.key, bucket, awsLocation, - this.test.copyKey, bucket, azureLocation, - this.test.copyKey, 'REPLACE', null, done); + assert.strictEqual(result.CopyObjectResult.ETag, `"${normalMD5}"`); + assertGetObjects( + this.test.key, + bucket, + awsLocation, + this.test.copyKey, + bucket, + azureLocation, + this.test.copyKey, + 'REPLACE', + null, + done, + ); }) - .catch(err => done(new Error( - `Expected success but got error: ${err}`))); + .catch(err => done(new Error(`Expected success but got error: ${err}`))); }); }); @@ -296,269 +339,347 @@ function testSuite() { }; s3.send(new CopyObjectCommand(copyParams)) .then(result => { - assert.strictEqual(result.CopyObjectResult.ETag, - `"${normalMD5}"`); - assertGetObjects(this.test.key, bucket, azureLocation, - this.test.copyKey, bucket, awsLocation, - this.test.key, 'REPLACE', null, done); + assert.strictEqual(result.CopyObjectResult.ETag, `"${normalMD5}"`); + assertGetObjects( + this.test.key, + bucket, + azureLocation, + this.test.copyKey, + bucket, + awsLocation, + this.test.key, + 'REPLACE', + null, + done, + ); }) - .catch(err => done(new Error( - `Expected success but got error: ${err}`))); + .catch(err => done(new Error(`Expected success but got error: ${err}`))); }); }); - it('should copy an object from Azure to mem with "REPLACE" directive ' + - 'and no location constraint md', function itFn(done) { - putSourceObj(this.test.key, azureLocation, null, bucket, () => { - const copyParams = { - Bucket: bucket, - Key: this.test.copyKey, - CopySource: `/${bucket}/${this.test.key}`, - MetadataDirective: 'REPLACE', - }; - s3.send(new CopyObjectCommand(copyParams)) - .then(result => { - assert.strictEqual(result.CopyObjectResult.ETag, - `"${normalMD5}"`); - assertGetObjects(this.test.key, bucket, azureLocation, - this.test.copyKey, bucket, undefined, - this.test.key, 'REPLACE', null, done); - }) - .catch(err => done(new Error( - `Expected success but got error: ${err}`))); - }); - }); + it( + 'should copy an object from Azure to mem with "REPLACE" directive ' + 'and no location constraint md', + function itFn(done) { + putSourceObj(this.test.key, azureLocation, null, bucket, () => { + const copyParams = { + Bucket: bucket, + Key: this.test.copyKey, + CopySource: `/${bucket}/${this.test.key}`, + MetadataDirective: 'REPLACE', + }; + s3.send(new CopyObjectCommand(copyParams)) + .then(result => { + assert.strictEqual(result.CopyObjectResult.ETag, `"${normalMD5}"`); + assertGetObjects( + this.test.key, + bucket, + azureLocation, + this.test.copyKey, + bucket, + undefined, + this.test.key, + 'REPLACE', + null, + done, + ); + }) + .catch(err => done(new Error(`Expected success but got error: ${err}`))); + }); + }, + ); - it('should copy an object from mem to Azure with "REPLACE" directive ' + - 'and no location constraint md', function itFn(done) { - putSourceObj(this.test.key, null, null, bucket, () => { - const copyParams = { - Bucket: bucketAzure, - Key: this.test.copyKey, - CopySource: `/${bucket}/${this.test.key}`, - MetadataDirective: 'REPLACE', - }; - s3.send(new CopyObjectCommand(copyParams)) - .then(result => { - assert.strictEqual(result.CopyObjectResult.ETag, - `"${normalMD5}"`); - assertGetObjects(this.test.key, bucket, undefined, - this.test.copyKey, bucketAzure, undefined, - this.test.copyKey, 'REPLACE', null, done); - }) - .catch(err => done(new Error( - `Expected success but got error: ${err}`))); - }); - }); + it( + 'should copy an object from mem to Azure with "REPLACE" directive ' + 'and no location constraint md', + function itFn(done) { + putSourceObj(this.test.key, null, null, bucket, () => { + const copyParams = { + Bucket: bucketAzure, + Key: this.test.copyKey, + CopySource: `/${bucket}/${this.test.key}`, + MetadataDirective: 'REPLACE', + }; + s3.send(new CopyObjectCommand(copyParams)) + .then(result => { + assert.strictEqual(result.CopyObjectResult.ETag, `"${normalMD5}"`); + assertGetObjects( + this.test.key, + bucket, + undefined, + this.test.copyKey, + bucketAzure, + undefined, + this.test.copyKey, + 'REPLACE', + null, + done, + ); + }) + .catch(err => done(new Error(`Expected success but got error: ${err}`))); + }); + }, + ); - it('should copy an object from Azure to Azure showing sending ' + - 'metadata location constraint this doesn\'t matter with COPY directive', - function itFn(done) { - putSourceObj(this.test.key, azureLocation, null, bucketAzure, - () => { - const copyParams = { - Bucket: bucketAzure, - Key: this.test.copyKey, - CopySource: `/${bucketAzure}/${this.test.key}`, - MetadataDirective: 'COPY', - Metadata: { 'scal-location-constraint': memLocation }, - }; - s3.send(new CopyObjectCommand(copyParams)) - .then(result => { - assert.strictEqual(result.CopyObjectResult.ETag, - `"${normalMD5}"`); - assertGetObjects(this.test.key, bucketAzure, - azureLocation, - this.test.copyKey, bucketAzure, azureLocation, - this.test.copyKey, 'COPY', null, done); - }) - .catch(err => done(new Error( - `Expected success but got error: ${err}`))); - }); - }); + it( + 'should copy an object from Azure to Azure showing sending ' + + "metadata location constraint this doesn't matter with COPY directive", + function itFn(done) { + putSourceObj(this.test.key, azureLocation, null, bucketAzure, () => { + const copyParams = { + Bucket: bucketAzure, + Key: this.test.copyKey, + CopySource: `/${bucketAzure}/${this.test.key}`, + MetadataDirective: 'COPY', + Metadata: { 'scal-location-constraint': memLocation }, + }; + s3.send(new CopyObjectCommand(copyParams)) + .then(result => { + assert.strictEqual(result.CopyObjectResult.ETag, `"${normalMD5}"`); + assertGetObjects( + this.test.key, + bucketAzure, + azureLocation, + this.test.copyKey, + bucketAzure, + azureLocation, + this.test.copyKey, + 'COPY', + null, + done, + ); + }) + .catch(err => done(new Error(`Expected success but got error: ${err}`))); + }); + }, + ); - it('should copy an object with no location constraint from Azure to ' + - 'Azure relying on the bucket location constraint', - function itFn(done) { - putSourceObj(this.test.key, null, null, bucketAzure, - () => { - const copyParams = { - Bucket: bucketAzure, - Key: this.test.copyKey, - CopySource: `/${bucketAzure}/${this.test.key}`, - MetadataDirective: 'COPY', - }; - s3.send(new CopyObjectCommand(copyParams)) - .then(result => { - assert.strictEqual(result.CopyObjectResult.ETag, - `"${normalMD5}"`); - assertGetObjects(this.test.key, bucketAzure, - undefined, this.test.copyKey, bucketAzure, - undefined, this.test.copyKey, 'COPY', null, done); - }) - .catch(err => done(new Error( - `Expected success but got error: ${err}`))); - }); - }); + it( + 'should copy an object with no location constraint from Azure to ' + + 'Azure relying on the bucket location constraint', + function itFn(done) { + putSourceObj(this.test.key, null, null, bucketAzure, () => { + const copyParams = { + Bucket: bucketAzure, + Key: this.test.copyKey, + CopySource: `/${bucketAzure}/${this.test.key}`, + MetadataDirective: 'COPY', + }; + s3.send(new CopyObjectCommand(copyParams)) + .then(result => { + assert.strictEqual(result.CopyObjectResult.ETag, `"${normalMD5}"`); + assertGetObjects( + this.test.key, + bucketAzure, + undefined, + this.test.copyKey, + bucketAzure, + undefined, + this.test.copyKey, + 'COPY', + null, + done, + ); + }) + .catch(err => done(new Error(`Expected success but got error: ${err}`))); + }); + }, + ); - it('should copy an object from Azure to mem because bucket ' + - 'destination location is mem', function itFn(done) { - putSourceObj(this.test.key, azureLocation, null, bucket, () => { - const copyParams = { - Bucket: bucket, - Key: this.test.copyKey, - CopySource: `/${bucket}/${this.test.key}`, - MetadataDirective: 'COPY', - }; - s3.send(new CopyObjectCommand(copyParams)) - .then(result => { - assert.strictEqual(result.CopyObjectResult.ETag, - `"${normalMD5}"`); - assertGetObjects(this.test.key, bucket, azureLocation, - this.test.copyKey, bucket, memLocation, - this.test.key, 'COPY', null, done); - }) - .catch(err => done(new Error( - `Expected success but got error: ${err}`))); - }); - }); + it( + 'should copy an object from Azure to mem because bucket ' + 'destination location is mem', + function itFn(done) { + putSourceObj(this.test.key, azureLocation, null, bucket, () => { + const copyParams = { + Bucket: bucket, + Key: this.test.copyKey, + CopySource: `/${bucket}/${this.test.key}`, + MetadataDirective: 'COPY', + }; + s3.send(new CopyObjectCommand(copyParams)) + .then(result => { + assert.strictEqual(result.CopyObjectResult.ETag, `"${normalMD5}"`); + assertGetObjects( + this.test.key, + bucket, + azureLocation, + this.test.copyKey, + bucket, + memLocation, + this.test.key, + 'COPY', + null, + done, + ); + }) + .catch(err => done(new Error(`Expected success but got error: ${err}`))); + }); + }, + ); - it('should copy an object on Azure to a different Azure ' + - 'account without source object READ access', - function itFn(done) { - putSourceObj(this.test.key, azureLocation2, null, bucket, () => { - const copyParams = { - Bucket: bucket, - Key: this.test.copyKey, - CopySource: `/${bucket}/${this.test.key}`, - MetadataDirective: 'REPLACE', - Metadata: { 'scal-location-constraint': azureLocation }, - }; - s3.send(new CopyObjectCommand(copyParams)) - .then(result => { - assert.strictEqual(result.CopyObjectResult.ETag, - `"${normalMD5}"`); - assertGetObjects(this.test.key, bucket, - azureLocation2, this.test.copyKey, bucket, - azureLocation, this.test.copyKey, 'REPLACE', null, - done); - }) - .catch(err => done(new Error( - `Expected success but got error: ${err}`))); - }); - }); + it( + 'should copy an object on Azure to a different Azure ' + 'account without source object READ access', + function itFn(done) { + putSourceObj(this.test.key, azureLocation2, null, bucket, () => { + const copyParams = { + Bucket: bucket, + Key: this.test.copyKey, + CopySource: `/${bucket}/${this.test.key}`, + MetadataDirective: 'REPLACE', + Metadata: { 'scal-location-constraint': azureLocation }, + }; + s3.send(new CopyObjectCommand(copyParams)) + .then(result => { + assert.strictEqual(result.CopyObjectResult.ETag, `"${normalMD5}"`); + assertGetObjects( + this.test.key, + bucket, + azureLocation2, + this.test.copyKey, + bucket, + azureLocation, + this.test.copyKey, + 'REPLACE', + null, + done, + ); + }) + .catch(err => done(new Error(`Expected success but got error: ${err}`))); + }); + }, + ); - it('should copy a 5MB object on Azure to a different Azure ' + - 'account without source object READ access', - function itFn(done) { - putSourceObj(this.test.key, azureLocation2, { big: true }, bucket, - () => { - const copyParams = { - Bucket: bucket, - Key: this.test.copyKey, - CopySource: `/${bucket}/${this.test.key}`, - MetadataDirective: 'REPLACE', - Metadata: { 'scal-location-constraint': azureLocation }, - }; - s3.send(new CopyObjectCommand(copyParams)) - .then(result => { - assert.strictEqual(result.CopyObjectResult.ETag, - `"${bigMD5}"`); - assertGetObjects(this.test.key, bucket, - azureLocation2, this.test.copyKey, bucket, - azureLocation, this.test.copyKey, 'REPLACE', - { big: true }, done); - }) - .catch(err => done(new Error( - `Expected success but got error: ${err}`))); - }); - }); + it( + 'should copy a 5MB object on Azure to a different Azure ' + 'account without source object READ access', + function itFn(done) { + putSourceObj(this.test.key, azureLocation2, { big: true }, bucket, () => { + const copyParams = { + Bucket: bucket, + Key: this.test.copyKey, + CopySource: `/${bucket}/${this.test.key}`, + MetadataDirective: 'REPLACE', + Metadata: { 'scal-location-constraint': azureLocation }, + }; + s3.send(new CopyObjectCommand(copyParams)) + .then(result => { + assert.strictEqual(result.CopyObjectResult.ETag, `"${bigMD5}"`); + assertGetObjects( + this.test.key, + bucket, + azureLocation2, + this.test.copyKey, + bucket, + azureLocation, + this.test.copyKey, + 'REPLACE', + { big: true }, + done, + ); + }) + .catch(err => done(new Error(`Expected success but got error: ${err}`))); + }); + }, + ); - it('should copy an object from bucketmatch=false ' + - 'Azure location to MPU with a bucketmatch=false Azure location', - function itFn(done) { - putSourceObj(this.test.key, azureLocationMismatch, null, bucket, - () => { - const copyParams = { - Bucket: bucket, - Key: this.test.copyKey, - CopySource: `/${bucket}/${this.test.key}`, - MetadataDirective: 'REPLACE', - Metadata: { 'scal-location-constraint': - azureLocationMismatch }, - }; - s3.send(new CopyObjectCommand(copyParams)) - .then(result => { - assert.strictEqual(result.CopyObjectResult.ETag, - `"${normalMD5}"`); - assertGetObjects(this.test.key, bucket, - azureLocationMismatch, - this.test.copyKey, bucket, azureLocationMismatch, - `${bucket}/${this.test.copyKey}`, 'REPLACE', null, - done); - }) - .catch(err => done(new Error( - `Expected success but got error: ${err}`))); - }); - }); + it( + 'should copy an object from bucketmatch=false ' + + 'Azure location to MPU with a bucketmatch=false Azure location', + function itFn(done) { + putSourceObj(this.test.key, azureLocationMismatch, null, bucket, () => { + const copyParams = { + Bucket: bucket, + Key: this.test.copyKey, + CopySource: `/${bucket}/${this.test.key}`, + MetadataDirective: 'REPLACE', + Metadata: { 'scal-location-constraint': azureLocationMismatch }, + }; + s3.send(new CopyObjectCommand(copyParams)) + .then(result => { + assert.strictEqual(result.CopyObjectResult.ETag, `"${normalMD5}"`); + assertGetObjects( + this.test.key, + bucket, + azureLocationMismatch, + this.test.copyKey, + bucket, + azureLocationMismatch, + `${bucket}/${this.test.copyKey}`, + 'REPLACE', + null, + done, + ); + }) + .catch(err => done(new Error(`Expected success but got error: ${err}`))); + }); + }, + ); - it('should copy an object from bucketmatch=false ' + - 'Azure location to MPU with a bucketmatch=true Azure location', - function itFn(done) { - putSourceObj(this.test.key, azureLocationMismatch, null, bucket, - () => { - const copyParams = { - Bucket: bucket, - Key: this.test.copyKey, - CopySource: `/${bucket}/${this.test.key}`, - MetadataDirective: 'REPLACE', - Metadata: { 'scal-location-constraint': azureLocation }, - }; - s3.send(new CopyObjectCommand(copyParams)) - .then(result => { - assert.strictEqual(result.CopyObjectResult.ETag, - `"${normalMD5}"`); - assertGetObjects(this.test.key, bucket, - azureLocationMismatch, - this.test.copyKey, bucket, azureLocation, - this.test.copyKey, 'REPLACE', null, done); - }) - .catch(err => done(new Error( - `Expected success but got error: ${err}`))); - }); - }); + it( + 'should copy an object from bucketmatch=false ' + + 'Azure location to MPU with a bucketmatch=true Azure location', + function itFn(done) { + putSourceObj(this.test.key, azureLocationMismatch, null, bucket, () => { + const copyParams = { + Bucket: bucket, + Key: this.test.copyKey, + CopySource: `/${bucket}/${this.test.key}`, + MetadataDirective: 'REPLACE', + Metadata: { 'scal-location-constraint': azureLocation }, + }; + s3.send(new CopyObjectCommand(copyParams)) + .then(result => { + assert.strictEqual(result.CopyObjectResult.ETag, `"${normalMD5}"`); + assertGetObjects( + this.test.key, + bucket, + azureLocationMismatch, + this.test.copyKey, + bucket, + azureLocation, + this.test.copyKey, + 'REPLACE', + null, + done, + ); + }) + .catch(err => done(new Error(`Expected success but got error: ${err}`))); + }); + }, + ); - it('should copy an object from bucketmatch=true ' + - 'Azure location to MPU with a bucketmatch=false Azure location', - function itFn(done) { - putSourceObj(this.test.key, azureLocation, null, bucket, () => { - const copyParams = { - Bucket: bucket, - Key: this.test.copyKey, - CopySource: `/${bucket}/${this.test.key}`, - MetadataDirective: 'REPLACE', - Metadata: { 'scal-location-constraint': - azureLocationMismatch }, - }; - s3.send(new CopyObjectCommand(copyParams)) - .then(result => { - assert.strictEqual(result.CopyObjectResult.ETag, - `"${normalMD5}"`); - assertGetObjects(this.test.key, bucket, - azureLocation, - this.test.copyKey, bucket, azureLocationMismatch, - `${bucket}/${this.test.copyKey}`, - 'REPLACE', null, done); - }) - .catch(err => done(new Error( - `Expected success but got error: ${err}`))); - }); - }); + it( + 'should copy an object from bucketmatch=true ' + + 'Azure location to MPU with a bucketmatch=false Azure location', + function itFn(done) { + putSourceObj(this.test.key, azureLocation, null, bucket, () => { + const copyParams = { + Bucket: bucket, + Key: this.test.copyKey, + CopySource: `/${bucket}/${this.test.key}`, + MetadataDirective: 'REPLACE', + Metadata: { 'scal-location-constraint': azureLocationMismatch }, + }; + s3.send(new CopyObjectCommand(copyParams)) + .then(result => { + assert.strictEqual(result.CopyObjectResult.ETag, `"${normalMD5}"`); + assertGetObjects( + this.test.key, + bucket, + azureLocation, + this.test.copyKey, + bucket, + azureLocationMismatch, + `${bucket}/${this.test.copyKey}`, + 'REPLACE', + null, + done, + ); + }) + .catch(err => done(new Error(`Expected success but got error: ${err}`))); + }); + }, + ); - it('should copy a 0-byte object from mem to Azure', - function itFn(done) { - putSourceObj(this.test.key, memLocation, { empty: true }, bucket, - () => { + it('should copy a 0-byte object from mem to Azure', function itFn(done) { + putSourceObj(this.test.key, memLocation, { empty: true }, bucket, () => { const copyParams = { Bucket: bucket, Key: this.test.copyKey, @@ -568,21 +689,26 @@ function testSuite() { }; s3.send(new CopyObjectCommand(copyParams)) .then(result => { - assert.strictEqual(result.CopyObjectResult.ETag, - `"${emptyMD5}"`); - assertGetObjects(this.test.key, bucket, memLocation, - this.test.copyKey, bucket, azureLocation, - this.test.copyKey, 'REPLACE', { empty: true }, - done); + assert.strictEqual(result.CopyObjectResult.ETag, `"${emptyMD5}"`); + assertGetObjects( + this.test.key, + bucket, + memLocation, + this.test.copyKey, + bucket, + azureLocation, + this.test.copyKey, + 'REPLACE', + { empty: true }, + done, + ); }) - .catch(err => done(new Error( - `Expected success but got error: ${err}`))); + .catch(err => done(new Error(`Expected success but got error: ${err}`))); }); }); it('should copy a 0-byte object on Azure', function itFn(done) { - putSourceObj(this.test.key, azureLocation, { empty: true }, bucket, - () => { + putSourceObj(this.test.key, azureLocation, { empty: true }, bucket, () => { const copyParams = { Bucket: bucket, Key: this.test.copyKey, @@ -592,21 +718,26 @@ function testSuite() { }; s3.send(new CopyObjectCommand(copyParams)) .then(result => { - assert.strictEqual(result.CopyObjectResult.ETag, - `"${emptyMD5}"`); - assertGetObjects(this.test.key, bucket, azureLocation, - this.test.copyKey, bucket, azureLocation, - this.test.copyKey, 'REPLACE', { empty: true }, - done); + assert.strictEqual(result.CopyObjectResult.ETag, `"${emptyMD5}"`); + assertGetObjects( + this.test.key, + bucket, + azureLocation, + this.test.copyKey, + bucket, + azureLocation, + this.test.copyKey, + 'REPLACE', + { empty: true }, + done, + ); }) - .catch(err => done(new Error( - `Expected success but got error: ${err}`))); + .catch(err => done(new Error(`Expected success but got error: ${err}`))); }); }); it('should copy a 5MB object from mem to Azure', function itFn(done) { - putSourceObj(this.test.key, memLocation, { big: true }, bucket, - () => { + putSourceObj(this.test.key, memLocation, { big: true }, bucket, () => { const copyParams = { Bucket: bucket, Key: this.test.copyKey, @@ -616,24 +747,28 @@ function testSuite() { }; s3.send(new CopyObjectCommand(copyParams)) .then(result => { - assert.strictEqual(result.CopyObjectResult.ETag, - `"${bigMD5}"`); + assert.strictEqual(result.CopyObjectResult.ETag, `"${bigMD5}"`); setTimeout(() => { - assertGetObjects(this.test.key, bucket, + assertGetObjects( + this.test.key, + bucket, memLocation, - this.test.copyKey, bucket, azureLocation, - this.test.copyKey, 'REPLACE', - { big: true }, done); + this.test.copyKey, + bucket, + azureLocation, + this.test.copyKey, + 'REPLACE', + { big: true }, + done, + ); }, azureTimeout); }) - .catch(err => done(new Error( - `Err copying object: ${err}`))); + .catch(err => done(new Error(`Err copying object: ${err}`))); }); }); it('should copy a 5MB object on Azure', function itFn(done) { - putSourceObj(this.test.key, azureLocation, { big: true }, bucket, - () => { + putSourceObj(this.test.key, azureLocation, { big: true }, bucket, () => { const copyParams = { Bucket: bucket, Key: this.test.copyKey, @@ -643,29 +778,30 @@ function testSuite() { }; s3.send(new CopyObjectCommand(copyParams)) .then(result => { - assert.strictEqual(result.CopyObjectResult.ETag, - `"${bigMD5}"`); + assert.strictEqual(result.CopyObjectResult.ETag, `"${bigMD5}"`); setTimeout(() => { - assertGetObjects(this.test.key, bucket, + assertGetObjects( + this.test.key, + bucket, + azureLocation, + this.test.copyKey, + bucket, azureLocation, - this.test.copyKey, bucket, azureLocation, - this.test.copyKey, 'REPLACE', - { big: true }, done); + this.test.copyKey, + 'REPLACE', + { big: true }, + done, + ); }, azureTimeout); }) - .catch(err => done(new Error( - `Err copying object: ${err}`))); + .catch(err => done(new Error(`Err copying object: ${err}`))); }); }); - it('should return error if Azure source object has ' + - 'been deleted', function itFn(done) { - putSourceObj(this.test.key, azureLocation, null, bucket, - () => { - azureClient.deleteBlob(azureContainerName, this.test.key, - err => { - assert.equal(err, null, 'Error deleting object from ' + - `Azure: ${err}`); + it('should return error if Azure source object has ' + 'been deleted', function itFn(done) { + putSourceObj(this.test.key, azureLocation, null, bucket, () => { + azureClient.deleteBlob(azureContainerName, this.test.key, err => { + assert.equal(err, null, 'Error deleting object from ' + `Azure: ${err}`); const copyParams = { Bucket: bucket, Key: this.test.copyKey, @@ -673,8 +809,7 @@ function testSuite() { MetadataDirective: 'COPY', }; s3.send(new CopyObjectCommand(copyParams)) - .then(() => done(new Error( - 'Expected ServiceUnavailable error'))) + .then(() => done(new Error('Expected ServiceUnavailable error'))) .catch(err => { assert.strictEqual(err.name, 'ServiceUnavailable'); done(); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/objectCopy/objectCopy.js b/tests/functional/aws-node-sdk/test/multipleBackend/objectCopy/objectCopy.js index 4cece36110..d1984b596c 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/objectCopy/objectCopy.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/objectCopy/objectCopy.js @@ -1,6 +1,6 @@ const assert = require('assert'); -const { - S3Client, +const { + S3Client, PutObjectCommand, GetObjectCommand, CopyObjectCommand, @@ -13,11 +13,18 @@ const BucketUtility = require('../../../lib/utility/bucket-util'); const constants = require('../../../../../../constants'); const { config } = require('../../../../../../lib/Config'); const { getRealAwsConfig } = require('../../support/awsConfig'); -const { createEncryptedBucketPromise } = - require('../../../lib/utility/createEncryptedBucket'); -const { describeSkipIfNotMultiple, awsS3, memLocation, awsLocation, - azureLocation, awsLocation2, awsLocationMismatch, awsLocationEncryption, - genUniqID } = require('../utils'); +const { createEncryptedBucketPromise } = require('../../../lib/utility/createEncryptedBucket'); +const { + describeSkipIfNotMultiple, + awsS3, + memLocation, + awsLocation, + azureLocation, + awsLocation2, + awsLocationMismatch, + awsLocationEncryption, + genUniqID, +} = require('../utils'); const bucket = `objectcopybucket${genUniqID()}`; const bucketAws = `objectcopyaws${genUniqID()}`; @@ -32,7 +39,9 @@ let s3; async function putSourceObj(location, isEmptyObj, bucket) { const key = `somekey-${genUniqID()}`; - const sourceParams = { Bucket: bucket, Key: key, + const sourceParams = { + Bucket: bucket, + Key: key, Metadata: { 'test-header': 'copyme', }, @@ -53,94 +62,105 @@ async function putSourceObj(location, isEmptyObj, bucket) { return key; } -async function assertGetObjects(sourceKey, sourceBucket, sourceLoc, destKey, -destBucket, destLoc, awsKey, mdDirective, isEmptyObj, awsS3, awsLocation) { - const awsBucket = - config.locationConstraints[awsLocation].details.bucketName; +async function assertGetObjects( + sourceKey, + sourceBucket, + sourceLoc, + destKey, + destBucket, + destLoc, + awsKey, + mdDirective, + isEmptyObj, + awsS3, + awsLocation, +) { + const awsBucket = config.locationConstraints[awsLocation].details.bucketName; const sourceGetParams = { Bucket: sourceBucket, Key: sourceKey }; const destGetParams = { Bucket: destBucket, Key: destKey }; const awsParams = { Bucket: awsBucket, Key: awsKey }; - + const [sourceRes, destRes, awsRes] = await Promise.all([ s3.send(new GetObjectCommand(sourceGetParams)), s3.send(new GetObjectCommand(destGetParams)), awsS3.send(new GetObjectCommand(awsParams)), ]); - if (isEmptyObj) { - assert.strictEqual(sourceRes.ETag, `"${emptyMD5}"`); - assert.strictEqual(destRes.ETag, `"${emptyMD5}"`); - assert.strictEqual(awsRes.ETag, `"${emptyMD5}"`); - } else if (process.env.ENABLE_KMS_ENCRYPTION === 'true') { - assert.strictEqual(sourceRes.ServerSideEncryption, 'AES256'); - assert.strictEqual(destRes.ServerSideEncryption, 'AES256'); - } else { - assert.strictEqual(sourceRes.ETag, `"${correctMD5}"`); - assert.strictEqual(destRes.ETag, `"${correctMD5}"`); - assert.deepStrictEqual(sourceRes.Body, destRes.Body); - assert.strictEqual(awsRes.ETag, `"${correctMD5}"`); - assert.deepStrictEqual(sourceRes.Body, awsRes.Body); - } - if (destLoc === awsLocationEncryption) { - assert.strictEqual(awsRes.ServerSideEncryption, 'AES256'); - } else { - assert.strictEqual(awsRes.ServerSideEncryption, undefined); - } + if (isEmptyObj) { + assert.strictEqual(sourceRes.ETag, `"${emptyMD5}"`); + assert.strictEqual(destRes.ETag, `"${emptyMD5}"`); + assert.strictEqual(awsRes.ETag, `"${emptyMD5}"`); + } else if (process.env.ENABLE_KMS_ENCRYPTION === 'true') { + assert.strictEqual(sourceRes.ServerSideEncryption, 'AES256'); + assert.strictEqual(destRes.ServerSideEncryption, 'AES256'); + } else { + assert.strictEqual(sourceRes.ETag, `"${correctMD5}"`); + assert.strictEqual(destRes.ETag, `"${correctMD5}"`); + assert.deepStrictEqual(sourceRes.Body, destRes.Body); + assert.strictEqual(awsRes.ETag, `"${correctMD5}"`); + assert.deepStrictEqual(sourceRes.Body, awsRes.Body); + } + if (destLoc === awsLocationEncryption) { + assert.strictEqual(awsRes.ServerSideEncryption, 'AES256'); + } else { + assert.strictEqual(awsRes.ServerSideEncryption, undefined); + } + if (mdDirective === 'COPY') { + assert.deepStrictEqual(sourceRes.Metadata['test-header'], destRes.Metadata['test-header']); + } else if (mdDirective === 'REPLACE') { + assert.strictEqual(destRes.Metadata['test-header'], undefined); + } + if (destLoc === awsLocation) { + assert.strictEqual(awsRes.Metadata[locMetaHeader], destLoc); if (mdDirective === 'COPY') { - assert.deepStrictEqual(sourceRes.Metadata['test-header'], - destRes.Metadata['test-header']); + assert.deepStrictEqual(sourceRes.Metadata['test-header'], awsRes.Metadata['test-header']); } else if (mdDirective === 'REPLACE') { - assert.strictEqual(destRes.Metadata['test-header'], - undefined); - } - if (destLoc === awsLocation) { - assert.strictEqual(awsRes.Metadata[locMetaHeader], destLoc); - if (mdDirective === 'COPY') { - assert.deepStrictEqual(sourceRes.Metadata['test-header'], - awsRes.Metadata['test-header']); - } else if (mdDirective === 'REPLACE') { - assert.strictEqual(awsRes.Metadata['test-header'], - undefined); - } + assert.strictEqual(awsRes.Metadata['test-header'], undefined); } + } assert.strictEqual(sourceRes.ContentLength, destRes.ContentLength); assert.strictEqual(sourceRes.Metadata[locMetaHeader], sourceLoc); assert.strictEqual(destRes.Metadata[locMetaHeader], destLoc); } -describeSkipIfNotMultiple('MultipleBackend object copy: AWS', -function testSuite() { +describeSkipIfNotMultiple('MultipleBackend object copy: AWS', function testSuite() { this.timeout(250000); withV4(sigCfg => { beforeEach(async () => { bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; process.stdout.write('Creating bucket\n'); - + if (process.env.ENABLE_KMS_ENCRYPTION === 'true') { await createEncryptedBucketPromise({ Bucket: bucket }); await createEncryptedBucketPromise({ Bucket: awsServerSideEncryptionbucket }); await createEncryptedBucketPromise({ Bucket: bucketAws }); } else { - await s3.send(new CreateBucketCommand({ - Bucket: bucket, - CreateBucketConfiguration: { - LocationConstraint: memLocation, - }, - })); - - await s3.send(new CreateBucketCommand({ - Bucket: awsServerSideEncryptionbucket, - CreateBucketConfiguration: { - LocationConstraint: awsLocationEncryption, - }, - })); - - await s3.send(new CreateBucketCommand({ - Bucket: bucketAws, - CreateBucketConfiguration: { - LocationConstraint: awsLocation, - }, - })); + await s3.send( + new CreateBucketCommand({ + Bucket: bucket, + CreateBucketConfiguration: { + LocationConstraint: memLocation, + }, + }), + ); + + await s3.send( + new CreateBucketCommand({ + Bucket: awsServerSideEncryptionbucket, + CreateBucketConfiguration: { + LocationConstraint: awsLocationEncryption, + }, + }), + ); + + await s3.send( + new CreateBucketCommand({ + Bucket: bucketAws, + CreateBucketConfiguration: { + LocationConstraint: awsLocation, + }, + }), + ); } }); @@ -154,9 +174,7 @@ function testSuite() { await bucketUtil.deleteOne(bucketAws); }); - it('should copy an object from mem to AWS relying on ' + - 'destination bucket location', - async () => { + it('should copy an object from mem to AWS relying on ' + 'destination bucket location', async () => { const key = await putSourceObj(memLocation, false, bucket); const copyKey = `copyKey-${genUniqID()}`; const copyParams = { @@ -168,14 +186,22 @@ function testSuite() { process.stdout.write('Copying object\n'); const result = await s3.send(new CopyObjectCommand(copyParams)); assert.strictEqual(result.CopyObjectResult.ETag, `"${correctMD5}"`); - await assertGetObjects(key, bucket, memLocation, copyKey, - bucketAws, awsLocation, copyKey, 'COPY', false, awsS3, - awsLocation); + await assertGetObjects( + key, + bucket, + memLocation, + copyKey, + bucketAws, + awsLocation, + copyKey, + 'COPY', + false, + awsS3, + awsLocation, + ); }); - it('should copy an object from Azure to AWS relying on ' + - 'destination bucket location', - async () => { + it('should copy an object from Azure to AWS relying on ' + 'destination bucket location', async () => { const key = await putSourceObj(azureLocation, false, bucket); const copyKey = `copyKey-${genUniqID()}`; const copyParams = { @@ -187,33 +213,53 @@ function testSuite() { process.stdout.write('Copying object\n'); const result = await s3.send(new CopyObjectCommand(copyParams)); assert.strictEqual(result.CopyObjectResult.ETag, `"${correctMD5}"`); - await assertGetObjects(key, bucket, azureLocation, copyKey, - bucketAws, awsLocation, copyKey, 'COPY', false, awsS3, - awsLocation); + await assertGetObjects( + key, + bucket, + azureLocation, + copyKey, + bucketAws, + awsLocation, + copyKey, + 'COPY', + false, + awsS3, + awsLocation, + ); }); - it('should copy an object without location contraint from mem ' + - 'to AWS relying on destination bucket location', - async () => { - const key = await putSourceObj(null, false, bucket); - const copyKey = `copyKey-${genUniqID()}`; - const copyParams = { - Bucket: bucketAws, - Key: copyKey, - CopySource: `/${bucket}/${key}`, - MetadataDirective: 'COPY', - }; - process.stdout.write('Copying object\n'); - const result = await s3.send(new CopyObjectCommand(copyParams)); - assert.strictEqual(result.CopyObjectResult.ETag, `"${correctMD5}"`); - await assertGetObjects(key, bucket, undefined, copyKey, - bucketAws, undefined, copyKey, 'COPY', false, awsS3, - awsLocation); - }); + it( + 'should copy an object without location contraint from mem ' + + 'to AWS relying on destination bucket location', + async () => { + const key = await putSourceObj(null, false, bucket); + const copyKey = `copyKey-${genUniqID()}`; + const copyParams = { + Bucket: bucketAws, + Key: copyKey, + CopySource: `/${bucket}/${key}`, + MetadataDirective: 'COPY', + }; + process.stdout.write('Copying object\n'); + const result = await s3.send(new CopyObjectCommand(copyParams)); + assert.strictEqual(result.CopyObjectResult.ETag, `"${correctMD5}"`); + await assertGetObjects( + key, + bucket, + undefined, + copyKey, + bucketAws, + undefined, + copyKey, + 'COPY', + false, + awsS3, + awsLocation, + ); + }, + ); - it('should copy an object from AWS to mem relying on destination ' + - 'bucket location', - async () => { + it('should copy an object from AWS to mem relying on destination ' + 'bucket location', async () => { const key = await putSourceObj(awsLocation, false, bucketAws); const copyKey = `copyKey-${genUniqID()}`; const copyParams = { @@ -225,9 +271,19 @@ function testSuite() { process.stdout.write('Copying object\n'); const result = await s3.send(new CopyObjectCommand(copyParams)); assert.strictEqual(result.CopyObjectResult.ETag, `"${correctMD5}"`); - await assertGetObjects(key, bucketAws, awsLocation, copyKey, - bucket, memLocation, key, 'COPY', false, awsS3, - awsLocation); + await assertGetObjects( + key, + bucketAws, + awsLocation, + copyKey, + bucket, + memLocation, + key, + 'COPY', + false, + awsS3, + awsLocation, + ); }); it('should copy an object from mem to AWS', async () => { @@ -239,18 +295,28 @@ function testSuite() { CopySource: `/${bucket}/${key}`, MetadataDirective: 'REPLACE', Metadata: { - 'scal-location-constraint': awsLocation }, + 'scal-location-constraint': awsLocation, + }, }; process.stdout.write('Copying object\n'); const result = await s3.send(new CopyObjectCommand(copyParams)); assert.strictEqual(result.CopyObjectResult.ETag, `"${correctMD5}"`); - await assertGetObjects(key, bucket, memLocation, copyKey, bucket, - awsLocation, copyKey, 'REPLACE', false, awsS3, - awsLocation); + await assertGetObjects( + key, + bucket, + memLocation, + copyKey, + bucket, + awsLocation, + copyKey, + 'REPLACE', + false, + awsS3, + awsLocation, + ); }); - it('should copy an object from mem to AWS with aws server ' + - 'side encryption', async () => { + it('should copy an object from mem to AWS with aws server ' + 'side encryption', async () => { const key = await putSourceObj(memLocation, false, bucket); const copyKey = `copyKey-${genUniqID()}`; const copyParams = { @@ -259,37 +325,59 @@ function testSuite() { CopySource: `/${bucket}/${key}`, MetadataDirective: 'REPLACE', Metadata: { - 'scal-location-constraint': awsLocationEncryption }, + 'scal-location-constraint': awsLocationEncryption, + }, }; process.stdout.write('Copying object\n'); const result = await s3.send(new CopyObjectCommand(copyParams)); assert.strictEqual(result.CopyObjectResult.ETag, `"${correctMD5}"`); - await assertGetObjects(key, bucket, memLocation, copyKey, bucket, - awsLocationEncryption, copyKey, 'REPLACE', false, - awsS3, awsLocation); + await assertGetObjects( + key, + bucket, + memLocation, + copyKey, + bucket, + awsLocationEncryption, + copyKey, + 'REPLACE', + false, + awsS3, + awsLocation, + ); }); - it('should copy an object from AWS to mem with encryption with ' + - 'REPLACE directive but no location constraint', async () => { - const key = await putSourceObj(awsLocation, false, bucket); - const copyKey = `copyKey-${genUniqID()}`; - const copyParams = { - Bucket: bucket, - Key: copyKey, - CopySource: `/${bucket}/${key}`, - MetadataDirective: 'REPLACE', - }; - process.stdout.write('Copying object\n'); - const result = await s3.send(new CopyObjectCommand(copyParams)); - assert.strictEqual(result.CopyObjectResult.ETag, `"${correctMD5}"`); - await assertGetObjects(key, bucket, awsLocation, copyKey, bucket, - undefined, key, 'REPLACE', false, - awsS3, awsLocation); - }); + it( + 'should copy an object from AWS to mem with encryption with ' + + 'REPLACE directive but no location constraint', + async () => { + const key = await putSourceObj(awsLocation, false, bucket); + const copyKey = `copyKey-${genUniqID()}`; + const copyParams = { + Bucket: bucket, + Key: copyKey, + CopySource: `/${bucket}/${key}`, + MetadataDirective: 'REPLACE', + }; + process.stdout.write('Copying object\n'); + const result = await s3.send(new CopyObjectCommand(copyParams)); + assert.strictEqual(result.CopyObjectResult.ETag, `"${correctMD5}"`); + await assertGetObjects( + key, + bucket, + awsLocation, + copyKey, + bucket, + undefined, + key, + 'REPLACE', + false, + awsS3, + awsLocation, + ); + }, + ); - it('should copy an object on AWS with aws server side ' + - 'encryption', - async () => { + it('should copy an object on AWS with aws server side ' + 'encryption', async () => { const key = await putSourceObj(awsLocation, false, bucket); const copyKey = `copyKey-${genUniqID()}`; const copyParams = { @@ -298,18 +386,28 @@ function testSuite() { CopySource: `/${bucket}/${key}`, MetadataDirective: 'REPLACE', Metadata: { - 'scal-location-constraint': awsLocationEncryption }, + 'scal-location-constraint': awsLocationEncryption, + }, }; process.stdout.write('Copying object\n'); const result = await s3.send(new CopyObjectCommand(copyParams)); assert.strictEqual(result.CopyObjectResult.ETag, `"${correctMD5}"`); - await assertGetObjects(key, bucket, awsLocation, copyKey, bucket, - awsLocationEncryption, copyKey, 'REPLACE', false, awsS3, - awsLocation); + await assertGetObjects( + key, + bucket, + awsLocation, + copyKey, + bucket, + awsLocationEncryption, + copyKey, + 'REPLACE', + false, + awsS3, + awsLocation, + ); }); - it('should copy an object on AWS with aws server side ' + - 'encrypted bucket', async () => { + it('should copy an object on AWS with aws server side ' + 'encrypted bucket', async () => { const key = await putSourceObj(awsLocation, false, awsServerSideEncryptionbucket); const copyKey = `copyKey-${genUniqID()}`; const copyParams = { @@ -321,33 +419,53 @@ function testSuite() { process.stdout.write('Copying object\n'); const result = await s3.send(new CopyObjectCommand(copyParams)); assert.strictEqual(result.CopyObjectResult.ETag, `"${correctMD5}"`); - await assertGetObjects(key, awsServerSideEncryptionbucket, - awsLocation, copyKey, awsServerSideEncryptionbucket, - awsLocationEncryption, copyKey, 'COPY', - false, awsS3, awsLocation); + await assertGetObjects( + key, + awsServerSideEncryptionbucket, + awsLocation, + copyKey, + awsServerSideEncryptionbucket, + awsLocationEncryption, + copyKey, + 'COPY', + false, + awsS3, + awsLocation, + ); }); - it('should copy an object from mem to AWS with encryption with ' + - 'REPLACE directive but no location constraint', async () => { - const key = await putSourceObj(null, false, bucket); - const copyKey = `copyKey-${genUniqID()}`; - const copyParams = { - Bucket: bucketAws, - Key: copyKey, - CopySource: `/${bucket}/${key}`, - MetadataDirective: 'REPLACE', - }; - process.stdout.write('Copying object\n'); - const result = await s3.send(new CopyObjectCommand(copyParams)); - assert.strictEqual(result.CopyObjectResult.ETag, `"${correctMD5}"`); - await assertGetObjects(key, bucket, undefined, copyKey, - bucketAws, undefined, copyKey, 'REPLACE', false, - awsS3, awsLocation); - }); + it( + 'should copy an object from mem to AWS with encryption with ' + + 'REPLACE directive but no location constraint', + async () => { + const key = await putSourceObj(null, false, bucket); + const copyKey = `copyKey-${genUniqID()}`; + const copyParams = { + Bucket: bucketAws, + Key: copyKey, + CopySource: `/${bucket}/${key}`, + MetadataDirective: 'REPLACE', + }; + process.stdout.write('Copying object\n'); + const result = await s3.send(new CopyObjectCommand(copyParams)); + assert.strictEqual(result.CopyObjectResult.ETag, `"${correctMD5}"`); + await assertGetObjects( + key, + bucket, + undefined, + copyKey, + bucketAws, + undefined, + copyKey, + 'REPLACE', + false, + awsS3, + awsLocation, + ); + }, + ); - it('should copy an object from AWS to mem with "COPY" ' + - 'directive and aws location metadata', - async () => { + it('should copy an object from AWS to mem with "COPY" ' + 'directive and aws location metadata', async () => { const key = await putSourceObj(awsLocation, false, bucket); const copyKey = `copyKey-${genUniqID()}`; const copyParams = { @@ -356,14 +474,25 @@ function testSuite() { CopySource: `/${bucket}/${key}`, MetadataDirective: 'COPY', Metadata: { - 'scal-location-constraint': awsLocation }, + 'scal-location-constraint': awsLocation, + }, }; process.stdout.write('Copying object\n'); const result = await s3.send(new CopyObjectCommand(copyParams)); assert.strictEqual(result.CopyObjectResult.ETag, `"${correctMD5}"`); - await assertGetObjects(key, bucket, awsLocation, copyKey, bucket, - memLocation, key, 'COPY', false, awsS3, - awsLocation); + await assertGetObjects( + key, + bucket, + awsLocation, + copyKey, + bucket, + memLocation, + key, + 'COPY', + false, + awsS3, + awsLocation, + ); }); it('should copy an object on AWS', async () => { @@ -379,88 +508,124 @@ function testSuite() { process.stdout.write('Copying object\n'); const result = await s3.send(new CopyObjectCommand(copyParams)); assert.strictEqual(result.CopyObjectResult.ETag, `"${correctMD5}"`); - await assertGetObjects(key, bucket, awsLocation, copyKey, bucket, - awsLocation, copyKey, 'REPLACE', false, awsS3, - awsLocation); + await assertGetObjects( + key, + bucket, + awsLocation, + copyKey, + bucket, + awsLocation, + copyKey, + 'REPLACE', + false, + awsS3, + awsLocation, + ); }); - it('should copy an object on AWS location with bucketMatch equals ' + - 'false to a different AWS location with bucketMatch equals true', - async () => { - const key = await putSourceObj(awsLocationMismatch, false, bucket); - const copyKey = `copyKey-${genUniqID()}`; - const copyParams = { - Bucket: bucket, - Key: copyKey, - CopySource: `/${bucket}/${key}`, - MetadataDirective: 'REPLACE', - Metadata: { - 'scal-location-constraint': awsLocation }, - }; - process.stdout.write('Copying object\n'); - const result = await s3.send(new CopyObjectCommand(copyParams)); - assert.strictEqual(result.CopyObjectResult.ETag, `"${correctMD5}"`); - await assertGetObjects(key, bucket, awsLocationMismatch, copyKey, - bucket, awsLocation, copyKey, 'REPLACE', false, awsS3, - awsLocation); - }); + it( + 'should copy an object on AWS location with bucketMatch equals ' + + 'false to a different AWS location with bucketMatch equals true', + async () => { + const key = await putSourceObj(awsLocationMismatch, false, bucket); + const copyKey = `copyKey-${genUniqID()}`; + const copyParams = { + Bucket: bucket, + Key: copyKey, + CopySource: `/${bucket}/${key}`, + MetadataDirective: 'REPLACE', + Metadata: { + 'scal-location-constraint': awsLocation, + }, + }; + process.stdout.write('Copying object\n'); + const result = await s3.send(new CopyObjectCommand(copyParams)); + assert.strictEqual(result.CopyObjectResult.ETag, `"${correctMD5}"`); + await assertGetObjects( + key, + bucket, + awsLocationMismatch, + copyKey, + bucket, + awsLocation, + copyKey, + 'REPLACE', + false, + awsS3, + awsLocation, + ); + }, + ); - it('should copy an object on AWS to a different AWS location ' + - 'with source object READ access', - async () => { + it('should copy an object on AWS to a different AWS location ' + 'with source object READ access', async () => { const awsConfig2 = getRealAwsConfig(awsLocation2); const awsS3Two = new S3Client(awsConfig2); const copyKey = `copyKey-${genUniqID()}`; - const awsBucket = - config.locationConstraints[awsLocation].details.bucketName; - + const awsBucket = config.locationConstraints[awsLocation].details.bucketName; + // giving access to the object on the AWS side const key = await putSourceObj(awsLocation, false, bucket); - await awsS3.send(new PutObjectAclCommand({ - Bucket: awsBucket, - Key: key, - ACL: 'public-read' - })); - + await awsS3.send( + new PutObjectAclCommand({ + Bucket: awsBucket, + Key: key, + ACL: 'public-read', + }), + ); + const copyParams = { Bucket: bucket, Key: copyKey, CopySource: `/${bucket}/${key}`, MetadataDirective: 'REPLACE', Metadata: { - 'scal-location-constraint': awsLocation2 }, + 'scal-location-constraint': awsLocation2, + }, }; process.stdout.write('Copying object\n'); const result = await s3.send(new CopyObjectCommand(copyParams)); assert.strictEqual(result.CopyObjectResult.ETag, `"${correctMD5}"`); - - await assertGetObjects(key, bucket, awsLocation, copyKey, - bucket, awsLocation2, copyKey, 'REPLACE', false, - awsS3Two, awsLocation2); - }); - it('should return error AccessDenied copying an object on ' + - 'AWS to a different AWS account without source object READ access', - async () => { - const key = await putSourceObj(awsLocation, false, bucket); - const copyKey = `copyKey-${genUniqID()}`; - const copyParams = { - Bucket: bucket, - Key: copyKey, - CopySource: `/${bucket}/${key}`, - MetadataDirective: 'REPLACE', - Metadata: { - 'scal-location-constraint': awsLocation2 }, - }; - process.stdout.write('Copying object\n'); - try { - await s3.send(new CopyObjectCommand(copyParams)); - assert.fail('Expected AccessDenied error'); - } catch (err) { - assert.strictEqual(err.name, 'AccessDenied'); - } + await assertGetObjects( + key, + bucket, + awsLocation, + copyKey, + bucket, + awsLocation2, + copyKey, + 'REPLACE', + false, + awsS3Two, + awsLocation2, + ); }); + it( + 'should return error AccessDenied copying an object on ' + + 'AWS to a different AWS account without source object READ access', + async () => { + const key = await putSourceObj(awsLocation, false, bucket); + const copyKey = `copyKey-${genUniqID()}`; + const copyParams = { + Bucket: bucket, + Key: copyKey, + CopySource: `/${bucket}/${key}`, + MetadataDirective: 'REPLACE', + Metadata: { + 'scal-location-constraint': awsLocation2, + }, + }; + process.stdout.write('Copying object\n'); + try { + await s3.send(new CopyObjectCommand(copyParams)); + assert.fail('Expected AccessDenied error'); + } catch (err) { + assert.strictEqual(err.name, 'AccessDenied'); + } + }, + ); + it('should copy an object on AWS with REPLACE', async () => { const key = await putSourceObj(awsLocation, false, bucket); const copyKey = `copyKey-${genUniqID()}`; @@ -470,14 +635,25 @@ function testSuite() { CopySource: `/${bucket}/${key}`, MetadataDirective: 'REPLACE', Metadata: { - 'scal-location-constraint': awsLocation }, + 'scal-location-constraint': awsLocation, + }, }; process.stdout.write('Copying object\n'); const result = await s3.send(new CopyObjectCommand(copyParams)); assert.strictEqual(result.CopyObjectResult.ETag, `"${correctMD5}"`); - await assertGetObjects(key, bucket, awsLocation, copyKey, bucket, - awsLocation, copyKey, 'REPLACE', false, awsS3, - awsLocation); + await assertGetObjects( + key, + bucket, + awsLocation, + copyKey, + bucket, + awsLocation, + copyKey, + 'REPLACE', + false, + awsS3, + awsLocation, + ); }); it('should copy a 0-byte object from mem to AWS', async () => { @@ -489,14 +665,25 @@ function testSuite() { CopySource: `/${bucket}/${key}`, MetadataDirective: 'REPLACE', Metadata: { - 'scal-location-constraint': awsLocation }, + 'scal-location-constraint': awsLocation, + }, }; process.stdout.write('Copying object\n'); const result = await s3.send(new CopyObjectCommand(copyParams)); assert.strictEqual(result.CopyObjectResult.ETag, `"${emptyMD5}"`); - await assertGetObjects(key, bucket, memLocation, copyKey, bucket, - awsLocation, copyKey, 'REPLACE', true, awsS3, - awsLocation); + await assertGetObjects( + key, + bucket, + memLocation, + copyKey, + bucket, + awsLocation, + copyKey, + 'REPLACE', + true, + awsS3, + awsLocation, + ); }); it('should copy a 0-byte object on AWS', async () => { @@ -512,22 +699,30 @@ function testSuite() { process.stdout.write('Copying object\n'); const result = await s3.send(new CopyObjectCommand(copyParams)); assert.strictEqual(result.CopyObjectResult.ETag, `"${emptyMD5}"`); - await assertGetObjects(key, bucket, awsLocation, copyKey, bucket, - awsLocation, copyKey, 'REPLACE', true, awsS3, - awsLocation); + await assertGetObjects( + key, + bucket, + awsLocation, + copyKey, + bucket, + awsLocation, + copyKey, + 'REPLACE', + true, + awsS3, + awsLocation, + ); }); - it('should return error if AWS source object has ' + - 'been deleted', async () => { + it('should return error if AWS source object has ' + 'been deleted', async () => { const key = await putSourceObj(awsLocation, false, bucket); - const awsBucket = - config.locationConstraints[awsLocation].details.bucketName; - + const awsBucket = config.locationConstraints[awsLocation].details.bucketName; + await awsS3.send(new DeleteObjectCommand({ Bucket: awsBucket, Key: key })); - + const copyKey = `copyKey-${genUniqID()}`; - const copyParams = { - Bucket: bucket, + const copyParams = { + Bucket: bucket, Key: copyKey, CopySource: `/${bucket}/${key}`, MetadataDirective: 'REPLACE', diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/objectCopy/objectCopyAwsVersioning.js b/tests/functional/aws-node-sdk/test/multipleBackend/objectCopy/objectCopyAwsVersioning.js index 9a07b4d15e..ad5ab19959 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/objectCopy/objectCopyAwsVersioning.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/objectCopy/objectCopyAwsVersioning.js @@ -46,22 +46,24 @@ function _getCreateBucketParams(bucket, location) { } function createBuckets(testParams, cb) { - const { sourceBucket, sourceLocation, destBucket, destLocation } - = testParams; + const { sourceBucket, sourceLocation, destBucket, destLocation } = testParams; const sourceParams = _getCreateBucketParams(sourceBucket, sourceLocation); const destParams = _getCreateBucketParams(destBucket, destLocation); if (sourceBucket === destBucket) { - return s3.send(new CreateBucketCommand(sourceParams)) + return s3 + .send(new CreateBucketCommand(sourceParams)) .then(() => cb()) .catch(err => cb(err)); } - return async.map([sourceParams, destParams], + return async.map( + [sourceParams, destParams], (createParams, next) => { s3.send(new CreateBucketCommand(createParams)) .then(() => next()) .catch(next); }, - err => cb(err)); + err => cb(err), + ); } function putSourceObj(testParams, cb) { @@ -92,9 +94,16 @@ function putSourceObj(testParams, cb) { } function copyObject(testParams, cb) { - const { sourceBucket, sourceKey, sourceVersionId, sourceVersioningState, - destBucket, directive, destVersioningState, isEmptyObj } - = testParams; + const { + sourceBucket, + sourceKey, + sourceVersionId, + sourceVersioningState, + destBucket, + directive, + destVersioningState, + isEmptyObj, + } = testParams; const destKey = `destkey-${genUniqID()}`; const copyParams = { Bucket: destBucket, @@ -103,11 +112,9 @@ function copyObject(testParams, cb) { MetadataDirective: directive, }; if (sourceVersionId) { - copyParams.CopySource = - `${copyParams.CopySource}?versionId=${sourceVersionId}`; + copyParams.CopySource = `${copyParams.CopySource}?versionId=${sourceVersionId}`; } else if (sourceVersioningState === 'Suspended') { - copyParams.CopySource = - `${copyParams.CopySource}?versionId=null`; + copyParams.CopySource = `${copyParams.CopySource}?versionId=null`; } s3.send(new CopyObjectCommand(copyParams)) .then(data => { @@ -117,26 +124,24 @@ function copyObject(testParams, cb) { assert.strictEqual(data.VersionId, undefined); } const expectedBody = isEmptyObj ? '' : someBody; - return awsGetLatestVerId(destKey, expectedBody, - (err, awsVersionId) => { - if (err) { - cb(err); - return; - } - Object.assign(testParams, { - destKey, - destVersionId: data.VersionId, - awsVersionId, - }); - if (!data.VersionId && destVersioningState === 'Suspended') { - // eslint-disable-next-line no-param-reassign - testParams.destVersionId = 'null'; - } - cb(); + return awsGetLatestVerId(destKey, expectedBody, (err, awsVersionId) => { + if (err) { + cb(err); + return; + } + Object.assign(testParams, { + destKey, + destVersionId: data.VersionId, + awsVersionId, }); + if (!data.VersionId && destVersioningState === 'Suspended') { + // eslint-disable-next-line no-param-reassign + testParams.destVersionId = 'null'; + } + cb(); + }); }) - .catch(err => cb(new Error( - `Error copying object to destination: ${err}`))); + .catch(err => cb(new Error(`Error copying object to destination: ${err}`))); } async function loadBodyBuffer(res) { @@ -169,106 +174,108 @@ function assertGetObjects(testParams, cb) { isEmptyObj, directive, } = testParams; - const sourceGetParams = { Bucket: sourceBucket, Key: sourceKey, - VersionId: sourceVersionId }; - const destGetParams = { Bucket: destBucket, Key: destKey, - VersionId: destVersionId }; - const awsParams = { Bucket: awsBucket, Key: destKey, - VersionId: awsVersionId }; + const sourceGetParams = { Bucket: sourceBucket, Key: sourceKey, VersionId: sourceVersionId }; + const destGetParams = { Bucket: destBucket, Key: destKey, VersionId: destVersionId }; + const awsParams = { Bucket: awsBucket, Key: destKey, VersionId: awsVersionId }; - async.series([ - cb => { - s3.send(new GetObjectCommand(sourceGetParams)) - .then(async res => { - const bodyBuffer = await loadBodyBuffer(res); - if (bodyBuffer !== undefined) { - // eslint-disable-next-line no-param-reassign - res.Body = bodyBuffer; - } - cb(null, res); - }) - .catch(cb); - }, - cb => { - s3.send(new GetObjectCommand(destGetParams)) - .then(async res => { - const bodyBuffer = await loadBodyBuffer(res); - if (bodyBuffer !== undefined) { - // eslint-disable-next-line no-param-reassign - res.Body = bodyBuffer; - } - cb(null, res); - }) - .catch(cb); + async.series( + [ + cb => { + s3.send(new GetObjectCommand(sourceGetParams)) + .then(async res => { + const bodyBuffer = await loadBodyBuffer(res); + if (bodyBuffer !== undefined) { + // eslint-disable-next-line no-param-reassign + res.Body = bodyBuffer; + } + cb(null, res); + }) + .catch(cb); + }, + cb => { + s3.send(new GetObjectCommand(destGetParams)) + .then(async res => { + const bodyBuffer = await loadBodyBuffer(res); + if (bodyBuffer !== undefined) { + // eslint-disable-next-line no-param-reassign + res.Body = bodyBuffer; + } + cb(null, res); + }) + .catch(cb); + }, + cb => awsS3.getObject(awsParams, cb), + ], + (err, results) => { + assert.strictEqual(err, null, `Error in assertGetObjects: ${err}`); + const [sourceRes, destRes, awsRes] = results; + if (isEmptyObj) { + assert.strictEqual(sourceRes.ETag, `"${emptyMD5}"`); + assert.strictEqual(destRes.ETag, `"${emptyMD5}"`); + assert.strictEqual(awsRes.ETag, `"${emptyMD5}"`); + } else { + assert.strictEqual(sourceRes.ETag, `"${correctMD5}"`); + assert.strictEqual(destRes.ETag, `"${correctMD5}"`); + assert.deepStrictEqual(sourceRes.Body, destRes.Body); + assert.strictEqual(awsRes.ETag, `"${correctMD5}"`); + assert.deepStrictEqual(sourceRes.Body, awsRes.Body); + } + if (directive === 'COPY') { + assert.deepStrictEqual(sourceRes.Metadata, testMetadata); + assert.deepStrictEqual(sourceRes.Metadata, destRes.Metadata); + assert.deepStrictEqual(sourceRes.Metadata, awsRes.Metadata); + } else if (directive === 'REPLACE') { + assert.deepStrictEqual(destRes.Metadata, {}); + assert.deepStrictEqual(awsRes.Metadata, {}); + } + assert.strictEqual(sourceRes.ContentLength, destRes.ContentLength); + cb(); }, - cb => awsS3.getObject(awsParams, cb), - ], (err, results) => { - assert.strictEqual(err, null, `Error in assertGetObjects: ${err}`); - const [sourceRes, destRes, awsRes] = results; - if (isEmptyObj) { - assert.strictEqual(sourceRes.ETag, `"${emptyMD5}"`); - assert.strictEqual(destRes.ETag, `"${emptyMD5}"`); - assert.strictEqual(awsRes.ETag, `"${emptyMD5}"`); - } else { - assert.strictEqual(sourceRes.ETag, `"${correctMD5}"`); - assert.strictEqual(destRes.ETag, `"${correctMD5}"`); - assert.deepStrictEqual(sourceRes.Body, destRes.Body); - assert.strictEqual(awsRes.ETag, `"${correctMD5}"`); - assert.deepStrictEqual(sourceRes.Body, awsRes.Body); - } - if (directive === 'COPY') { - assert.deepStrictEqual(sourceRes.Metadata, testMetadata); - assert.deepStrictEqual(sourceRes.Metadata, destRes.Metadata); - assert.deepStrictEqual(sourceRes.Metadata, awsRes.Metadata); - } else if (directive === 'REPLACE') { - assert.deepStrictEqual(destRes.Metadata, {}); - assert.deepStrictEqual(awsRes.Metadata, {}); - } - assert.strictEqual(sourceRes.ContentLength, destRes.ContentLength); - cb(); - }); + ); } -describeSkipIfNotMultiple('AWS backend object copy with versioning', -function testSuite() { +describeSkipIfNotMultiple('AWS backend object copy with versioning', function testSuite() { this.timeout(250000); withV4(sigCfg => { bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; - afterEach(() => bucketUtil.empty(sourceBucketName) - .then(() => bucketUtil.deleteOne(sourceBucketName)) - .catch(err => { - process.stdout.write('Error deleting source bucket ' + - `in afterEach: ${err}\n`); - throw err; - }) - .then(() => bucketUtil.empty(destBucketName)) - .then(() => bucketUtil.deleteOne(destBucketName)) - .catch(err => { - if (err.code === 'NoSuchBucket') { - process.stdout.write('Warning: did not find dest bucket ' + - 'for deletion'); - // we do not throw err since dest bucket may not exist - // if we are using source as dest - } else { - process.stdout.write('Error deleting dest bucket ' + - `in afterEach: ${err}\n`); + afterEach(() => + bucketUtil + .empty(sourceBucketName) + .then(() => bucketUtil.deleteOne(sourceBucketName)) + .catch(err => { + process.stdout.write('Error deleting source bucket ' + `in afterEach: ${err}\n`); throw err; - } - }) + }) + .then(() => bucketUtil.empty(destBucketName)) + .then(() => bucketUtil.deleteOne(destBucketName)) + .catch(err => { + if (err.code === 'NoSuchBucket') { + process.stdout.write('Warning: did not find dest bucket ' + 'for deletion'); + // we do not throw err since dest bucket may not exist + // if we are using source as dest + } else { + process.stdout.write('Error deleting dest bucket ' + `in afterEach: ${err}\n`); + throw err; + } + }), ); - [{ - directive: 'REPLACE', - isEmptyObj: true, - }, { - directive: 'REPLACE', - isEmptyObj: false, - }, { - directive: 'COPY', - isEmptyObj: false, - }].forEach(testParams => { + [ + { + directive: 'REPLACE', + isEmptyObj: true, + }, + { + directive: 'REPLACE', + isEmptyObj: false, + }, + { + directive: 'COPY', + isEmptyObj: false, + }, + ].forEach(testParams => { Object.assign(testParams, { sourceBucket: sourceBucketName, sourceLocation: awsLocation, @@ -276,180 +283,215 @@ function testSuite() { destLocation: awsLocation, }); const { isEmptyObj, directive } = testParams; - it(`should copy ${isEmptyObj ? 'an empty' : ''} ` + - 'object from AWS backend non-versioned bucket' + - 'to AWS backend versioned bucket ' + - `with ${directive} directive`, done => { - Object.assign(testParams, { - sourceVersioningState: undefined, - destVersioningState: 'Enabled', - }); - async.waterfall([ - next => createBuckets(testParams, next), - next => putSourceObj(testParams, next), - next => enableVersioning(s3, testParams.destBucket, next), - next => copyObject(testParams, next), - // put another version to test and make sure version id from - // copy was stored to get the right version - next => putToAwsBackend(s3, destBucketName, - testParams.destKey, wrongVersionBody, () => next()), - next => assertGetObjects(testParams, next), - ], done); - }); + it( + `should copy ${isEmptyObj ? 'an empty' : ''} ` + + 'object from AWS backend non-versioned bucket' + + 'to AWS backend versioned bucket ' + + `with ${directive} directive`, + done => { + Object.assign(testParams, { + sourceVersioningState: undefined, + destVersioningState: 'Enabled', + }); + async.waterfall( + [ + next => createBuckets(testParams, next), + next => putSourceObj(testParams, next), + next => enableVersioning(s3, testParams.destBucket, next), + next => copyObject(testParams, next), + // put another version to test and make sure version id from + // copy was stored to get the right version + next => + putToAwsBackend(s3, destBucketName, testParams.destKey, wrongVersionBody, () => next()), + next => assertGetObjects(testParams, next), + ], + done, + ); + }, + ); - it(`should copy ${isEmptyObj ? 'an empty ' : ''}version ` + - 'from one AWS backend versioned bucket' + - `to another on ${directive} directive`, - done => { - Object.assign(testParams, { - sourceVersioningState: 'Enabled', - destVersioningState: 'Enabled', - }); - async.waterfall([ - next => createBuckets(testParams, next), - next => enableVersioning(s3, testParams.sourceBucket, next), - next => putSourceObj(testParams, next), - next => enableVersioning(s3, testParams.destBucket, next), - next => copyObject(testParams, next), - // put another version to test and make sure version id from - // copy was stored to get the right version - next => putToAwsBackend(s3, destBucketName, - testParams.destKey, wrongVersionBody, () => next()), - next => assertGetObjects(testParams, next), - ], done); - }); + it( + `should copy ${isEmptyObj ? 'an empty ' : ''}version ` + + 'from one AWS backend versioned bucket' + + `to another on ${directive} directive`, + done => { + Object.assign(testParams, { + sourceVersioningState: 'Enabled', + destVersioningState: 'Enabled', + }); + async.waterfall( + [ + next => createBuckets(testParams, next), + next => enableVersioning(s3, testParams.sourceBucket, next), + next => putSourceObj(testParams, next), + next => enableVersioning(s3, testParams.destBucket, next), + next => copyObject(testParams, next), + // put another version to test and make sure version id from + // copy was stored to get the right version + next => + putToAwsBackend(s3, destBucketName, testParams.destKey, wrongVersionBody, () => next()), + next => assertGetObjects(testParams, next), + ], + done, + ); + }, + ); - it(`should copy ${isEmptyObj ? 'an empty ' : ''}null ` + - 'version from one AWS backend versioning suspended bucket to ' + - ` another versioning suspended bucket with ${directive} directive`, - done => { - Object.assign(testParams, { - sourceVersioningState: 'Suspended', - destVersioningState: 'Suspended', - }); - async.waterfall([ - next => createBuckets(testParams, next), - next => suspendVersioning(s3, testParams.sourceBucket, - next), - next => putSourceObj(testParams, next), - next => suspendVersioning(s3, testParams.destBucket, next), - next => copyObject(testParams, next), - next => enableVersioning(s3, testParams.destBucket, next), - // put another version to test and make sure version id from - // copy was stored to get the right version - next => putToAwsBackend(s3, destBucketName, - testParams.destKey, wrongVersionBody, () => next()), - next => assertGetObjects(testParams, next), - ], done); - }); + it( + `should copy ${isEmptyObj ? 'an empty ' : ''}null ` + + 'version from one AWS backend versioning suspended bucket to ' + + ` another versioning suspended bucket with ${directive} directive`, + done => { + Object.assign(testParams, { + sourceVersioningState: 'Suspended', + destVersioningState: 'Suspended', + }); + async.waterfall( + [ + next => createBuckets(testParams, next), + next => suspendVersioning(s3, testParams.sourceBucket, next), + next => putSourceObj(testParams, next), + next => suspendVersioning(s3, testParams.destBucket, next), + next => copyObject(testParams, next), + next => enableVersioning(s3, testParams.destBucket, next), + // put another version to test and make sure version id from + // copy was stored to get the right version + next => + putToAwsBackend(s3, destBucketName, testParams.destKey, wrongVersionBody, () => next()), + next => assertGetObjects(testParams, next), + ], + done, + ); + }, + ); - it(`should copy ${isEmptyObj ? 'an empty ' : ''}version ` + - 'from a AWS backend versioned bucket to a versioned-suspended' + - `one with ${directive} directive`, done => { - Object.assign(testParams, { - sourceVersioningState: 'Enabled', - destVersioningState: 'Suspended', - }); - async.waterfall([ - next => createBuckets(testParams, next), - next => enableVersioning(s3, testParams.sourceBucket, next), - next => putSourceObj(testParams, next), - next => suspendVersioning(s3, testParams.destBucket, next), - next => copyObject(testParams, next), - // put another version to test and make sure version id from - // copy was stored to get the right version - next => enableVersioning(s3, testParams.destBucket, next), - next => putToAwsBackend(s3, destBucketName, - testParams.destKey, wrongVersionBody, () => next()), - next => assertGetObjects(testParams, next), - ], done); - }); + it( + `should copy ${isEmptyObj ? 'an empty ' : ''}version ` + + 'from a AWS backend versioned bucket to a versioned-suspended' + + `one with ${directive} directive`, + done => { + Object.assign(testParams, { + sourceVersioningState: 'Enabled', + destVersioningState: 'Suspended', + }); + async.waterfall( + [ + next => createBuckets(testParams, next), + next => enableVersioning(s3, testParams.sourceBucket, next), + next => putSourceObj(testParams, next), + next => suspendVersioning(s3, testParams.destBucket, next), + next => copyObject(testParams, next), + // put another version to test and make sure version id from + // copy was stored to get the right version + next => enableVersioning(s3, testParams.destBucket, next), + next => + putToAwsBackend(s3, destBucketName, testParams.destKey, wrongVersionBody, () => next()), + next => assertGetObjects(testParams, next), + ], + done, + ); + }, + ); }); - it('versioning not configured: if copy object to a ' + - 'pre-existing object on AWS backend, metadata should be overwritten ' + - 'but data of previous version in AWS should not be deleted', - function itF(done) { - const destKey = `destkey-${genUniqID()}`; - const testParams = { - sourceBucket: sourceBucketName, - sourceLocation: awsLocation, - sourceVersioningState: undefined, - destBucket: sourceBucketName, - destLocation: awsLocation, - destVersioningState: undefined, - isEmptyObj: true, - directive: 'REPLACE', - }; - async.waterfall([ - next => createBuckets(testParams, next), - next => putToAwsBackend(s3, testParams.destBucket, destKey, - someBody, err => next(err)), - next => awsGetLatestVerId(destKey, someBody, next), - (awsVerId, next) => { - this.test.awsVerId = awsVerId; - next(); - }, - next => putSourceObj(testParams, next), - next => { - s3.send(new CopyObjectCommand({ - Bucket: testParams.destBucket, - Key: destKey, - CopySource: `/${testParams.sourceBucket}` + - `/${testParams.sourceKey}`, - MetadataDirective: testParams.directive, - Metadata: { - 'scal-location-constraint': - testParams.destLocation, + it( + 'versioning not configured: if copy object to a ' + + 'pre-existing object on AWS backend, metadata should be overwritten ' + + 'but data of previous version in AWS should not be deleted', + function itF(done) { + const destKey = `destkey-${genUniqID()}`; + const testParams = { + sourceBucket: sourceBucketName, + sourceLocation: awsLocation, + sourceVersioningState: undefined, + destBucket: sourceBucketName, + destLocation: awsLocation, + destVersioningState: undefined, + isEmptyObj: true, + directive: 'REPLACE', + }; + async.waterfall( + [ + next => createBuckets(testParams, next), + next => putToAwsBackend(s3, testParams.destBucket, destKey, someBody, err => next(err)), + next => awsGetLatestVerId(destKey, someBody, next), + (awsVerId, next) => { + this.test.awsVerId = awsVerId; + next(); }, - })) - .then(res => next(null, res)) - .catch(next); - }, - (copyResult, next) => awsGetLatestVerId(destKey, '', - (err, awsVersionId) => { - testParams.destKey = destKey; - testParams.destVersionId = copyResult.VersionId; - testParams.awsVersionId = awsVersionId; - next(); - }), - next => { - s3.send(new DeleteObjectCommand({ - Bucket: testParams.destBucket, - Key: testParams.destKey, - VersionId: 'null', - })) - .then(res => next(null, res)) - .catch(next); - }, - (delData, next) => getAndAssertResult(s3, { bucket: - testParams.destBucket, key: testParams.destKey, - expectedError: 'NoSuchKey' }, next), - next => awsGetLatestVerId(testParams.destKey, someBody, next), - (awsVerId, next) => { - assert.strictEqual(awsVerId, this.test.awsVerId); - next(); - }, - ], done); - }); + next => putSourceObj(testParams, next), + next => { + s3.send( + new CopyObjectCommand({ + Bucket: testParams.destBucket, + Key: destKey, + CopySource: `/${testParams.sourceBucket}` + `/${testParams.sourceKey}`, + MetadataDirective: testParams.directive, + Metadata: { + 'scal-location-constraint': testParams.destLocation, + }, + }), + ) + .then(res => next(null, res)) + .catch(next); + }, + (copyResult, next) => + awsGetLatestVerId(destKey, '', (err, awsVersionId) => { + testParams.destKey = destKey; + testParams.destVersionId = copyResult.VersionId; + testParams.awsVersionId = awsVersionId; + next(); + }), + next => { + s3.send( + new DeleteObjectCommand({ + Bucket: testParams.destBucket, + Key: testParams.destKey, + VersionId: 'null', + }), + ) + .then(res => next(null, res)) + .catch(next); + }, + (delData, next) => + getAndAssertResult( + s3, + { bucket: testParams.destBucket, key: testParams.destKey, expectedError: 'NoSuchKey' }, + next, + ), + next => awsGetLatestVerId(testParams.destKey, someBody, next), + (awsVerId, next) => { + assert.strictEqual(awsVerId, this.test.awsVerId); + next(); + }, + ], + done, + ); + }, + ); - [{ - sourceLocation: memLocation, - directive: 'REPLACE', - isEmptyObj: true, - }, { - sourceLocation: fileLocation, - directive: 'REPLACE', - isEmptyObj: true, - }, { - sourceLocation: memLocation, - directive: 'COPY', - isEmptyObj: false, - }, { - sourceLocation: fileLocation, - directive: 'COPY', - isEmptyObj: false, - }].forEach(testParams => { + [ + { + sourceLocation: memLocation, + directive: 'REPLACE', + isEmptyObj: true, + }, + { + sourceLocation: fileLocation, + directive: 'REPLACE', + isEmptyObj: true, + }, + { + sourceLocation: memLocation, + directive: 'COPY', + isEmptyObj: false, + }, + { + sourceLocation: fileLocation, + directive: 'COPY', + isEmptyObj: false, + }, + ].forEach(testParams => { Object.assign(testParams, { sourceBucket: sourceBucketName, sourceVersioningState: 'Enabled', @@ -459,36 +501,48 @@ function testSuite() { }); const { sourceLocation, directive, isEmptyObj } = testParams; - it(`should copy ${isEmptyObj ? 'empty ' : ''}object from ` + - `${sourceLocation} to bucket on AWS backend with ` + - `versioning with ${directive}`, done => { - async.waterfall([ - next => createBuckets(testParams, next), - next => putSourceObj(testParams, next), - next => enableVersioning(s3, testParams.destBucket, next), - next => copyObject(testParams, next), - next => assertGetObjects(testParams, next), - ], done); - }); + it( + `should copy ${isEmptyObj ? 'empty ' : ''}object from ` + + `${sourceLocation} to bucket on AWS backend with ` + + `versioning with ${directive}`, + done => { + async.waterfall( + [ + next => createBuckets(testParams, next), + next => putSourceObj(testParams, next), + next => enableVersioning(s3, testParams.destBucket, next), + next => copyObject(testParams, next), + next => assertGetObjects(testParams, next), + ], + done, + ); + }, + ); - it(`should copy ${isEmptyObj ? 'an empty ' : ''}version from ` + - `${sourceLocation} to bucket on AWS backend with ` + - `versioning with ${directive} directive`, done => { - async.waterfall([ - next => createBuckets(testParams, next), - next => enableVersioning(s3, testParams.sourceBucket, next), - // returns a version id which is added to testParams - // to be used in object copy - next => putSourceObj(testParams, next), - next => enableVersioning(s3, testParams.destBucket, next), - next => copyObject(testParams, next), - // put another version to test and make sure version id - // from copy was stored to get the right version - next => putToAwsBackend(s3, destBucketName, - testParams.destKey, wrongVersionBody, () => next()), - next => assertGetObjects(testParams, next), - ], done); - }); + it( + `should copy ${isEmptyObj ? 'an empty ' : ''}version from ` + + `${sourceLocation} to bucket on AWS backend with ` + + `versioning with ${directive} directive`, + done => { + async.waterfall( + [ + next => createBuckets(testParams, next), + next => enableVersioning(s3, testParams.sourceBucket, next), + // returns a version id which is added to testParams + // to be used in object copy + next => putSourceObj(testParams, next), + next => enableVersioning(s3, testParams.destBucket, next), + next => copyObject(testParams, next), + // put another version to test and make sure version id + // from copy was stored to get the right version + next => + putToAwsBackend(s3, destBucketName, testParams.destKey, wrongVersionBody, () => next()), + next => assertGetObjects(testParams, next), + ], + done, + ); + }, + ); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/objectPutCopyPart/objectPutCopyPartAzure.js b/tests/functional/aws-node-sdk/test/multipleBackend/objectPutCopyPart/objectPutCopyPartAzure.js index b41fb15530..11c17f074f 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/objectPutCopyPart/objectPutCopyPartAzure.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/objectPutCopyPart/objectPutCopyPartAzure.js @@ -16,19 +16,27 @@ const azureMpuUtils = s3middleware.azureHelper.mpuUtils; const { config } = require('../../../../../../lib/Config'); const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); -const { uniqName, getAzureClient, azureLocation, azureLocationMismatch, - memLocation, awsLocation, awsS3, getOwnerInfo, genUniqID, - describeSkipIfNotMultiple } - = require('../utils'); - +const { + uniqName, + getAzureClient, + azureLocation, + azureLocationMismatch, + memLocation, + awsLocation, + awsS3, + getOwnerInfo, + genUniqID, + describeSkipIfNotMultiple, +} = require('../utils'); let azureContainerName; -if (config.locationConstraints[azureLocation] && -config.locationConstraints[azureLocation].details && -config.locationConstraints[azureLocation].details.azureContainerName) { - azureContainerName = - config.locationConstraints[azureLocation].details.azureContainerName; +if ( + config.locationConstraints[azureLocation] && + config.locationConstraints[azureLocation].details && + config.locationConstraints[azureLocation].details.azureContainerName +) { + azureContainerName = config.locationConstraints[azureLocation].details.azureContainerName; } const memBucketName = `memputcopypartazure${genUniqID()}`; @@ -66,12 +74,8 @@ const result = { MaxParts: 1000, IsTruncated: false, Parts: [], - Initiator: - { ID: ownerID, - DisplayName: ownerDisplayName }, - Owner: - { DisplayName: ownerDisplayName, - ID: ownerID }, + Initiator: { ID: ownerID, DisplayName: ownerDisplayName }, + Owner: { DisplayName: ownerDisplayName, ID: ownerID }, StorageClass: 'STANDARD', }; @@ -79,8 +83,7 @@ let s3; let bucketUtil; function assertCopyPart(infos, cb) { - const { azureContainerName, mpuKeyNameAzure, uploadId, md5, - subPartSize } = infos; + const { azureContainerName, mpuKeyNameAzure, uploadId, md5, subPartSize } = infos; const resultCopy = JSON.parse(JSON.stringify(result)); resultCopy.Bucket = azureContainerName; resultCopy.Key = mpuKeyNameAzure; @@ -89,40 +92,52 @@ function assertCopyPart(infos, cb) { for (let i = 0; i < subPartSize.length; i++) { totalSize = totalSize + subPartSize[i]; } - async.waterfall([ - next => { - s3.send(new ListPartsCommand({ - Bucket: azureContainerName, - Key: mpuKeyNameAzure, - UploadId: uploadId, - })) - .then(res => { - resultCopy.Parts = - [{ PartNumber: 1, - LastModified: res.Parts[0].LastModified, - ETag: `"${md5}"`, - Size: totalSize }]; - assert.deepStrictEqual(res, resultCopy); - next(); - }) - .catch(err => next(new Error( - `listParts: Expected success, got error: ${err}`))); - }, - next => azureClient.getContainerClient(azureContainerName) - .getBlockBlobClient(mpuKeyNameAzure) - .getBlockList('all').then(res => { - subPartSize.forEach((size, index) => { - const partName = azureMpuUtils.getBlockId(uploadId, 1, index); - assert.strictEqual(res.uncommittedBlocks[index].name, partName); - assert.equal(res.uncommittedBlocks[index].size, size); - }); - next(); - }, err => { - assert.equal(err, null, 'listBlocks: Expected ' + - `success, got error: ${err}`); - next(); - }), - ], cb); + async.waterfall( + [ + next => { + s3.send( + new ListPartsCommand({ + Bucket: azureContainerName, + Key: mpuKeyNameAzure, + UploadId: uploadId, + }), + ) + .then(res => { + resultCopy.Parts = [ + { + PartNumber: 1, + LastModified: res.Parts[0].LastModified, + ETag: `"${md5}"`, + Size: totalSize, + }, + ]; + assert.deepStrictEqual(res, resultCopy); + next(); + }) + .catch(err => next(new Error(`listParts: Expected success, got error: ${err}`))); + }, + next => + azureClient + .getContainerClient(azureContainerName) + .getBlockBlobClient(mpuKeyNameAzure) + .getBlockList('all') + .then( + res => { + subPartSize.forEach((size, index) => { + const partName = azureMpuUtils.getBlockId(uploadId, 1, index); + assert.strictEqual(res.uncommittedBlocks[index].name, partName); + assert.equal(res.uncommittedBlocks[index].size, size); + }); + next(); + }, + err => { + assert.equal(err, null, 'listBlocks: Expected ' + `success, got error: ${err}`); + next(); + }, + ), + ], + cb, + ); } describeSkipIfNotMultiple('Put Copy Part to AZURE', function describeF() { @@ -135,39 +150,33 @@ describeSkipIfNotMultiple('Put Copy Part to AZURE', function describeF() { afterEach(() => { process.stdout.write('Emptying bucket\n'); - return bucketUtil.empty(azureContainerName) - .then(() => bucketUtil.empty(memBucketName)) - .then(() => { - process.stdout.write(`Deleting bucket ${azureContainerName}\n`); - return bucketUtil.deleteOne(azureContainerName); - }) - .then(() => { - process.stdout.write(`Deleting bucket ${memBucketName}\n`); - return bucketUtil.deleteOne(memBucketName); - }) - .catch(err => { - process.stdout.write(`Error in afterEach: ${err}\n`); - throw err; - }); + return bucketUtil + .empty(azureContainerName) + .then(() => bucketUtil.empty(memBucketName)) + .then(() => { + process.stdout.write(`Deleting bucket ${azureContainerName}\n`); + return bucketUtil.deleteOne(azureContainerName); + }) + .then(() => { + process.stdout.write(`Deleting bucket ${memBucketName}\n`); + return bucketUtil.deleteOne(memBucketName); + }) + .catch(err => { + process.stdout.write(`Error in afterEach: ${err}\n`); + throw err; + }); }); describe('Basic test: ', () => { beforeEach(function beF(done) { - this.currentTest.keyNameNormalAzure = - `normalazure${uniqName(keyObjectAzure)}`; - this.currentTest.keyNameNormalAzureMismatch = - `normalazuremismatch${uniqName(keyObjectAzure)}`; - - this.currentTest.keyNameFiveMbAzure = - `fivembazure${uniqName(keyObjectAzure)}`; - this.currentTest.keyNameFiveMbMem = - `fivembmem${uniqName(keyObjectMemory)}`; - - this.currentTest.mpuKeyNameAzure = - `mpukeyname${uniqName(keyObjectAzure)}`; - this.currentTest.mpuKeyNameMem = - `mpukeyname${uniqName(keyObjectMemory)}`; - this.currentTest.mpuKeyNameAWS = - `mpukeyname${uniqName(keyObjectAWS)}`; + this.currentTest.keyNameNormalAzure = `normalazure${uniqName(keyObjectAzure)}`; + this.currentTest.keyNameNormalAzureMismatch = `normalazuremismatch${uniqName(keyObjectAzure)}`; + + this.currentTest.keyNameFiveMbAzure = `fivembazure${uniqName(keyObjectAzure)}`; + this.currentTest.keyNameFiveMbMem = `fivembmem${uniqName(keyObjectMemory)}`; + + this.currentTest.mpuKeyNameAzure = `mpukeyname${uniqName(keyObjectAzure)}`; + this.currentTest.mpuKeyNameMem = `mpukeyname${uniqName(keyObjectMemory)}`; + this.currentTest.mpuKeyNameAWS = `mpukeyname${uniqName(keyObjectAWS)}`; const paramsAzure = { Bucket: azureContainerName, Key: this.currentTest.mpuKeyNameAzure, @@ -183,86 +192,109 @@ describeSkipIfNotMultiple('Put Copy Part to AZURE', function describeF() { Key: this.currentTest.mpuKeyNameAWS, Metadata: { 'scal-location-constraint': awsLocation }, }; - async.waterfall([ - next => { - s3.send(new CreateBucketCommand({ Bucket: azureContainerName })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new CreateBucketCommand({ Bucket: memBucketName })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new PutObjectCommand({ - Bucket: azureContainerName, - Key: this.currentTest.keyNameNormalAzure, - Body: normalBody, - Metadata: { 'scal-location-constraint': azureLocation }, - })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new PutObjectCommand({ - Bucket: azureContainerName, - Key: this.currentTest.keyNameNormalAzureMismatch, - Body: normalBody, - Metadata: { 'scal-location-constraint': - azureLocationMismatch }, - })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new PutObjectCommand({ - Bucket: azureContainerName, - Key: this.currentTest.keyNameFiveMbAzure, - Body: fiveMbBody, - Metadata: { 'scal-location-constraint': azureLocation }, - })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new PutObjectCommand({ - Bucket: azureContainerName, - Key: this.currentTest.keyNameFiveMbMem, - Body: fiveMbBody, - Metadata: { 'scal-location-constraint': memLocation }, - })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new CreateMultipartUploadCommand(paramsAzure)) - .then(res => { - this.currentTest.uploadId = res.UploadId; - next(); - }) - .catch(err => next(new Error( - `createMultipartUpload on Azure: Expected success, got error: ${err}`))); - }, - next => { - s3.send(new CreateMultipartUploadCommand(paramsMem)) - .then(res => { - this.currentTest.uploadIdMem = res.UploadId; - next(); - }) - .catch(err => next(new Error( - `createMultipartUpload in memory: Expected success, got error: ${err}`))); - }, - next => { - s3.send(new CreateMultipartUploadCommand(paramsAWS)) - .then(res => { - this.currentTest.uploadIdAWS = res.UploadId; - next(); - }) - .catch(err => next(new Error( - `createMultipartUpload on AWS: Expected success, got error: ${err}`))); - }, - ], done); + async.waterfall( + [ + next => { + s3.send(new CreateBucketCommand({ Bucket: azureContainerName })) + .then(() => next()) + .catch(next); + }, + next => { + s3.send(new CreateBucketCommand({ Bucket: memBucketName })) + .then(() => next()) + .catch(next); + }, + next => { + s3.send( + new PutObjectCommand({ + Bucket: azureContainerName, + Key: this.currentTest.keyNameNormalAzure, + Body: normalBody, + Metadata: { 'scal-location-constraint': azureLocation }, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + s3.send( + new PutObjectCommand({ + Bucket: azureContainerName, + Key: this.currentTest.keyNameNormalAzureMismatch, + Body: normalBody, + Metadata: { 'scal-location-constraint': azureLocationMismatch }, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + s3.send( + new PutObjectCommand({ + Bucket: azureContainerName, + Key: this.currentTest.keyNameFiveMbAzure, + Body: fiveMbBody, + Metadata: { 'scal-location-constraint': azureLocation }, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + s3.send( + new PutObjectCommand({ + Bucket: azureContainerName, + Key: this.currentTest.keyNameFiveMbMem, + Body: fiveMbBody, + Metadata: { 'scal-location-constraint': memLocation }, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + s3.send(new CreateMultipartUploadCommand(paramsAzure)) + .then(res => { + this.currentTest.uploadId = res.UploadId; + next(); + }) + .catch(err => + next( + new Error( + `createMultipartUpload on Azure: Expected success, got error: ${err}`, + ), + ), + ); + }, + next => { + s3.send(new CreateMultipartUploadCommand(paramsMem)) + .then(res => { + this.currentTest.uploadIdMem = res.UploadId; + next(); + }) + .catch(err => + next( + new Error( + `createMultipartUpload in memory: Expected success, got error: ${err}`, + ), + ), + ); + }, + next => { + s3.send(new CreateMultipartUploadCommand(paramsAWS)) + .then(res => { + this.currentTest.uploadIdAWS = res.UploadId; + next(); + }) + .catch(err => + next( + new Error(`createMultipartUpload on AWS: Expected success, got error: ${err}`), + ), + ); + }, + ], + done, + ); }); afterEach(async function afterEachF() { const paramsAzure = { @@ -285,283 +317,279 @@ describeSkipIfNotMultiple('Put Copy Part to AZURE', function describeF() { await s3.send(new AbortMultipartUploadCommand(params)); } }); - it('should copy small part from Azure to MPU with Azure location', - function ifF(done) { + it('should copy small part from Azure to MPU with Azure location', function ifF(done) { const params = { Bucket: azureContainerName, - CopySource: - `${azureContainerName}/${this.test.keyNameNormalAzure}`, + CopySource: `${azureContainerName}/${this.test.keyNameNormalAzure}`, Key: this.test.mpuKeyNameAzure, PartNumber: 1, UploadId: this.test.uploadId, }; - async.waterfall([ - next => { - s3.send(new UploadPartCopyCommand(params)) - .then(res => { - assert.strictEqual(res.ETag, `"${normalMD5}"`); - next(); - }) - .catch(err => next(new Error( - `uploadPartCopy: Expected success, got error: ${err}`))); - }, - next => { - const infos = { - azureContainerName, - mpuKeyNameAzure: this.test.mpuKeyNameAzure, - uploadId: this.test.uploadId, - md5: normalMD5, - subPartSize: [normalBodySize], - }; - assertCopyPart(infos, next); - }, - ], done); + async.waterfall( + [ + next => { + s3.send(new UploadPartCopyCommand(params)) + .then(res => { + assert.strictEqual(res.ETag, `"${normalMD5}"`); + next(); + }) + .catch(err => next(new Error(`uploadPartCopy: Expected success, got error: ${err}`))); + }, + next => { + const infos = { + azureContainerName, + mpuKeyNameAzure: this.test.mpuKeyNameAzure, + uploadId: this.test.uploadId, + md5: normalMD5, + subPartSize: [normalBodySize], + }; + assertCopyPart(infos, next); + }, + ], + done, + ); }); - it('should copy small part from Azure location with ' + - 'bucketMatch=false to MPU with Azure location', - function ifF(done) { - const params = { - Bucket: azureContainerName, - CopySource: - `${azureContainerName}/` + - `${this.test.keyNameNormalAzureMismatch}`, - Key: this.test.mpuKeyNameAzure, - PartNumber: 1, - UploadId: this.test.uploadId, - }; - async.waterfall([ - next => { - s3.send(new UploadPartCopyCommand(params)) - .then(res => { - assert.strictEqual(res.ETag, `"${normalMD5}"`); - next(); - }) - .catch(err => next(new Error( - `uploadPartCopy: Expected success, got error: ${err}`))); - }, - next => { - const infos = { - azureContainerName, - mpuKeyNameAzure: this.test.mpuKeyNameAzure, - uploadId: this.test.uploadId, - md5: normalMD5, - subPartSize: [normalBodySize], - }; - assertCopyPart(infos, next); - }, - ], done); - }); + it( + 'should copy small part from Azure location with ' + 'bucketMatch=false to MPU with Azure location', + function ifF(done) { + const params = { + Bucket: azureContainerName, + CopySource: `${azureContainerName}/` + `${this.test.keyNameNormalAzureMismatch}`, + Key: this.test.mpuKeyNameAzure, + PartNumber: 1, + UploadId: this.test.uploadId, + }; + async.waterfall( + [ + next => { + s3.send(new UploadPartCopyCommand(params)) + .then(res => { + assert.strictEqual(res.ETag, `"${normalMD5}"`); + next(); + }) + .catch(err => + next(new Error(`uploadPartCopy: Expected success, got error: ${err}`)), + ); + }, + next => { + const infos = { + azureContainerName, + mpuKeyNameAzure: this.test.mpuKeyNameAzure, + uploadId: this.test.uploadId, + md5: normalMD5, + subPartSize: [normalBodySize], + }; + assertCopyPart(infos, next); + }, + ], + done, + ); + }, + ); - it('should copy 5 Mb part from Azure to MPU with Azure location', - function ifF(done) { + it('should copy 5 Mb part from Azure to MPU with Azure location', function ifF(done) { const params = { Bucket: azureContainerName, - CopySource: - `${azureContainerName}/${this.test.keyNameFiveMbAzure}`, + CopySource: `${azureContainerName}/${this.test.keyNameFiveMbAzure}`, Key: this.test.mpuKeyNameAzure, PartNumber: 1, UploadId: this.test.uploadId, }; - async.waterfall([ - next => { - s3.send(new UploadPartCopyCommand(params)) - .then(res => { - assert.strictEqual(res.ETag, `"${fiveMbMD5}"`); - next(); - }) - .catch(err => next(new Error( - `uploadPartCopy: Expected success, got error: ${err}`))); - }, - next => { - const infos = { - azureContainerName, - mpuKeyNameAzure: this.test.mpuKeyNameAzure, - uploadId: this.test.uploadId, - md5: fiveMbMD5, - subPartSize: [fiveMB], - }; - assertCopyPart(infos, next); - }, - ], done); + async.waterfall( + [ + next => { + s3.send(new UploadPartCopyCommand(params)) + .then(res => { + assert.strictEqual(res.ETag, `"${fiveMbMD5}"`); + next(); + }) + .catch(err => next(new Error(`uploadPartCopy: Expected success, got error: ${err}`))); + }, + next => { + const infos = { + azureContainerName, + mpuKeyNameAzure: this.test.mpuKeyNameAzure, + uploadId: this.test.uploadId, + md5: fiveMbMD5, + subPartSize: [fiveMB], + }; + assertCopyPart(infos, next); + }, + ], + done, + ); }); - it('should copy part from Azure to MPU with memory location', - function ifF(done) { + it('should copy part from Azure to MPU with memory location', function ifF(done) { const params = { Bucket: memBucketName, - CopySource: - `${azureContainerName}/${this.test.keyNameNormalAzure}`, + CopySource: `${azureContainerName}/${this.test.keyNameNormalAzure}`, Key: this.test.mpuKeyNameMem, PartNumber: 1, UploadId: this.test.uploadIdMem, }; - async.waterfall([ - next => { - s3.send(new UploadPartCopyCommand(params)) - .then(res => { - assert.strictEqual(res.ETag, `"${normalMD5}"`); - next(); - }) - .catch(err => next(new Error( - `uploadPartCopy: Expected success, got error: ${err}`))); - }, - next => { - s3.send(new ListPartsCommand({ - Bucket: memBucketName, - Key: this.test.mpuKeyNameMem, - UploadId: this.test.uploadIdMem, - })) - .then(res => { - const resultCopy = - JSON.parse(JSON.stringify(result)); - resultCopy.Bucket = memBucketName; - resultCopy.Key = this.test.mpuKeyNameMem; - resultCopy.UploadId = this.test.uploadIdMem; - resultCopy.Parts = - [{ PartNumber: 1, - LastModified: res.Parts[0].LastModified, - ETag: `"${normalMD5}"`, - Size: normalBodySize }]; - assert.deepStrictEqual(res, resultCopy); - next(); - }) - .catch(err => next(new Error( - `listParts: Expected success, got error: ${err}`))); - }, - ], done); + async.waterfall( + [ + next => { + s3.send(new UploadPartCopyCommand(params)) + .then(res => { + assert.strictEqual(res.ETag, `"${normalMD5}"`); + next(); + }) + .catch(err => next(new Error(`uploadPartCopy: Expected success, got error: ${err}`))); + }, + next => { + s3.send( + new ListPartsCommand({ + Bucket: memBucketName, + Key: this.test.mpuKeyNameMem, + UploadId: this.test.uploadIdMem, + }), + ) + .then(res => { + const resultCopy = JSON.parse(JSON.stringify(result)); + resultCopy.Bucket = memBucketName; + resultCopy.Key = this.test.mpuKeyNameMem; + resultCopy.UploadId = this.test.uploadIdMem; + resultCopy.Parts = [ + { + PartNumber: 1, + LastModified: res.Parts[0].LastModified, + ETag: `"${normalMD5}"`, + Size: normalBodySize, + }, + ]; + assert.deepStrictEqual(res, resultCopy); + next(); + }) + .catch(err => next(new Error(`listParts: Expected success, got error: ${err}`))); + }, + ], + done, + ); }); - it('should copy part from Azure to MPU with AWS location', - function ifF(done) { + it('should copy part from Azure to MPU with AWS location', function ifF(done) { const params = { Bucket: memBucketName, - CopySource: - `${azureContainerName}/${this.test.keyNameNormalAzure}`, + CopySource: `${azureContainerName}/${this.test.keyNameNormalAzure}`, Key: this.test.mpuKeyNameAWS, PartNumber: 1, UploadId: this.test.uploadIdAWS, }; - async.waterfall([ - next => { - s3.send(new UploadPartCopyCommand(params)) - .then(res => { - assert.strictEqual(res.ETag, `"${normalMD5}"`); - next(); - }) - .catch(err => next(new Error( - `uploadPartCopy: Expected success, got error: ${err}`))); - }, - next => { - const awsBucket = - config.locationConstraints[awsLocation] - .details.bucketName; - awsS3.listParts({ - Bucket: awsBucket, - Key: this.test.mpuKeyNameAWS, - UploadId: this.test.uploadIdAWS, - }, (err, res) => { - assert.equal(err, null, - 'listParts: Expected success,' + - ` got error: ${err}`); - assert.strictEqual(res.Bucket, awsBucket); - assert.strictEqual(res.Key, - this.test.mpuKeyNameAWS); - assert.strictEqual(res.UploadId, - this.test.uploadIdAWS); - assert.strictEqual(res.Parts.length, 1); - assert.strictEqual(res.Parts[0].PartNumber, 1); - assert.strictEqual(res.Parts[0].ETag, - `"${normalMD5}"`); - assert.strictEqual(res.Parts[0].Size, - normalBodySize); - next(); - }); - }, - ], done); + async.waterfall( + [ + next => { + s3.send(new UploadPartCopyCommand(params)) + .then(res => { + assert.strictEqual(res.ETag, `"${normalMD5}"`); + next(); + }) + .catch(err => next(new Error(`uploadPartCopy: Expected success, got error: ${err}`))); + }, + next => { + const awsBucket = config.locationConstraints[awsLocation].details.bucketName; + awsS3.listParts( + { + Bucket: awsBucket, + Key: this.test.mpuKeyNameAWS, + UploadId: this.test.uploadIdAWS, + }, + (err, res) => { + assert.equal(err, null, 'listParts: Expected success,' + ` got error: ${err}`); + assert.strictEqual(res.Bucket, awsBucket); + assert.strictEqual(res.Key, this.test.mpuKeyNameAWS); + assert.strictEqual(res.UploadId, this.test.uploadIdAWS); + assert.strictEqual(res.Parts.length, 1); + assert.strictEqual(res.Parts[0].PartNumber, 1); + assert.strictEqual(res.Parts[0].ETag, `"${normalMD5}"`); + assert.strictEqual(res.Parts[0].Size, normalBodySize); + next(); + }, + ); + }, + ], + done, + ); }); - it('should copy part from Azure object with range to MPU ' + - 'with AWS location', function ifF(done) { + it('should copy part from Azure object with range to MPU ' + 'with AWS location', function ifF(done) { const params = { Bucket: memBucketName, - CopySource: - `${azureContainerName}/${this.test.keyNameNormalAzure}`, + CopySource: `${azureContainerName}/${this.test.keyNameNormalAzure}`, Key: this.test.mpuKeyNameAWS, CopySourceRange: 'bytes=0-5', PartNumber: 1, UploadId: this.test.uploadIdAWS, }; - async.waterfall([ - next => { - s3.send(new UploadPartCopyCommand(params)) - .then(res => { - assert.strictEqual(res.ETag, `"${sixBytesMD5}"`); - next(); - }) - .catch(err => next(new Error( - `uploadPartCopy: Expected success, got error: ${err}`))); - }, - next => { - const awsBucket = - config.locationConstraints[awsLocation] - .details.bucketName; - awsS3.listParts({ - Bucket: awsBucket, - Key: this.test.mpuKeyNameAWS, - UploadId: this.test.uploadIdAWS, - }, (err, res) => { - assert.equal(err, null, - 'listParts: Expected success,' + - ` got error: ${err}`); - assert.strictEqual(res.Bucket, awsBucket); - assert.strictEqual(res.Key, - this.test.mpuKeyNameAWS); - assert.strictEqual(res.UploadId, - this.test.uploadIdAWS); - assert.strictEqual(res.Parts.length, 1); - assert.strictEqual(res.Parts[0].PartNumber, 1); - assert.strictEqual(res.Parts[0].ETag, - `"${sixBytesMD5}"`); - assert.strictEqual(res.Parts[0].Size, 6); - next(); - }); - }, - ], done); + async.waterfall( + [ + next => { + s3.send(new UploadPartCopyCommand(params)) + .then(res => { + assert.strictEqual(res.ETag, `"${sixBytesMD5}"`); + next(); + }) + .catch(err => next(new Error(`uploadPartCopy: Expected success, got error: ${err}`))); + }, + next => { + const awsBucket = config.locationConstraints[awsLocation].details.bucketName; + awsS3.listParts( + { + Bucket: awsBucket, + Key: this.test.mpuKeyNameAWS, + UploadId: this.test.uploadIdAWS, + }, + (err, res) => { + assert.equal(err, null, 'listParts: Expected success,' + ` got error: ${err}`); + assert.strictEqual(res.Bucket, awsBucket); + assert.strictEqual(res.Key, this.test.mpuKeyNameAWS); + assert.strictEqual(res.UploadId, this.test.uploadIdAWS); + assert.strictEqual(res.Parts.length, 1); + assert.strictEqual(res.Parts[0].PartNumber, 1); + assert.strictEqual(res.Parts[0].ETag, `"${sixBytesMD5}"`); + assert.strictEqual(res.Parts[0].Size, 6); + next(); + }, + ); + }, + ], + done, + ); }); - it('should copy 5 Mb part from a memory location to MPU with ' + - 'Azure location', - function ifF(done) { + it('should copy 5 Mb part from a memory location to MPU with ' + 'Azure location', function ifF(done) { const params = { Bucket: azureContainerName, - CopySource: - `${azureContainerName}/${this.test.keyNameFiveMbMem}`, + CopySource: `${azureContainerName}/${this.test.keyNameFiveMbMem}`, Key: this.test.mpuKeyNameAzure, PartNumber: 1, UploadId: this.test.uploadId, }; - async.waterfall([ - next => { - s3.send(new UploadPartCopyCommand(params)) - .then(res => { - assert.strictEqual(res.ETag, `"${fiveMbMD5}"`); - next(); - }) - .catch(err => next(new Error( - `uploadPartCopy: Expected success, got error: ${err}`))); - }, - next => { - const infos = { - azureContainerName, - mpuKeyNameAzure: this.test.mpuKeyNameAzure, - uploadId: this.test.uploadId, - md5: fiveMbMD5, - subPartSize: [fiveMB], - }; - assertCopyPart(infos, next); - }, - ], done); + async.waterfall( + [ + next => { + s3.send(new UploadPartCopyCommand(params)) + .then(res => { + assert.strictEqual(res.ETag, `"${fiveMbMD5}"`); + next(); + }) + .catch(err => next(new Error(`uploadPartCopy: Expected success, got error: ${err}`))); + }, + next => { + const infos = { + azureContainerName, + mpuKeyNameAzure: this.test.mpuKeyNameAzure, + uploadId: this.test.uploadId, + md5: fiveMbMD5, + subPartSize: [fiveMB], + }; + assertCopyPart(infos, next); + }, + ], + done, + ); }); describe('with existing part', () => { @@ -577,79 +605,92 @@ describeSkipIfNotMultiple('Put Copy Part to AZURE', function describeF() { .then(() => done()) .catch(done); }); - it('should copy part from Azure to Azure with existing ' + - 'parts', function ifF(done) { + it('should copy part from Azure to Azure with existing ' + 'parts', function ifF(done) { const resultCopy = JSON.parse(JSON.stringify(result)); const params = { Bucket: azureContainerName, - CopySource: - `${azureContainerName}/${this.test.keyNameNormalAzure}`, + CopySource: `${azureContainerName}/${this.test.keyNameNormalAzure}`, Key: this.test.mpuKeyNameAzure, PartNumber: 2, UploadId: this.test.uploadId, }; - async.waterfall([ - next => { - s3.send(new UploadPartCopyCommand(params)) - .then(res => { - assert.strictEqual(res.ETag, `"${normalMD5}"`); - next(); - }) - .catch(err => next(new Error( - `uploadPartCopy: Expected success, got error: ${err}`))); - }, - next => { - s3.send(new ListPartsCommand({ - Bucket: azureContainerName, - Key: this.test.mpuKeyNameAzure, - UploadId: this.test.uploadId, - })) - .then(res => { - resultCopy.Bucket = azureContainerName; - resultCopy.Key = this.test.mpuKeyNameAzure; - resultCopy.UploadId = this.test.uploadId; - resultCopy.Parts = - [{ PartNumber: 1, - LastModified: res.Parts[0].LastModified, - ETag: `"${oneKbMD5}"`, - Size: oneKb }, - { PartNumber: 2, - LastModified: res.Parts[1].LastModified, - ETag: `"${normalMD5}"`, - Size: 11 }, - ]; - assert.deepStrictEqual(res, resultCopy); - next(); - }) - .catch(err => next(new Error( - `listParts: Expected success, got error: ${err}`))); - }, - next => azureClient.getContainerClient(azureContainerName) - .getBlockBlobClient(this.test.mpuKeyNameAzure) - .getBlockList('all').then(res => { - const partName = azureMpuUtils.getBlockId( - this.test.uploadId, 1, 0); - const partName2 = azureMpuUtils.getBlockId( - this.test.uploadId, 2, 0); - assert.strictEqual(res.uncommittedBlocks[0].name, partName); - assert.equal(res.uncommittedBlocks[0].size, oneKb); - assert.strictEqual(res.uncommittedBlocks[1].name, partName2); - assert.equal(res.uncommittedBlocks[1].size, 11); - next(); - }, err => { - assert.equal(err, null, 'listBlocks: Expected ' + - `success, got error: ${err}`); - next(); - }), - ], done); + async.waterfall( + [ + next => { + s3.send(new UploadPartCopyCommand(params)) + .then(res => { + assert.strictEqual(res.ETag, `"${normalMD5}"`); + next(); + }) + .catch(err => + next(new Error(`uploadPartCopy: Expected success, got error: ${err}`)), + ); + }, + next => { + s3.send( + new ListPartsCommand({ + Bucket: azureContainerName, + Key: this.test.mpuKeyNameAzure, + UploadId: this.test.uploadId, + }), + ) + .then(res => { + resultCopy.Bucket = azureContainerName; + resultCopy.Key = this.test.mpuKeyNameAzure; + resultCopy.UploadId = this.test.uploadId; + resultCopy.Parts = [ + { + PartNumber: 1, + LastModified: res.Parts[0].LastModified, + ETag: `"${oneKbMD5}"`, + Size: oneKb, + }, + { + PartNumber: 2, + LastModified: res.Parts[1].LastModified, + ETag: `"${normalMD5}"`, + Size: 11, + }, + ]; + assert.deepStrictEqual(res, resultCopy); + next(); + }) + .catch(err => next(new Error(`listParts: Expected success, got error: ${err}`))); + }, + next => + azureClient + .getContainerClient(azureContainerName) + .getBlockBlobClient(this.test.mpuKeyNameAzure) + .getBlockList('all') + .then( + res => { + const partName = azureMpuUtils.getBlockId(this.test.uploadId, 1, 0); + const partName2 = azureMpuUtils.getBlockId(this.test.uploadId, 2, 0); + assert.strictEqual(res.uncommittedBlocks[0].name, partName); + assert.equal(res.uncommittedBlocks[0].size, oneKb); + assert.strictEqual(res.uncommittedBlocks[1].name, partName2); + assert.equal(res.uncommittedBlocks[1].size, 11); + next(); + }, + err => { + assert.equal( + err, + null, + 'listBlocks: Expected ' + `success, got error: ${err}`, + ); + next(); + }, + ), + ], + done, + ); }); }); }); }); }); -describeSkipIfNotMultiple('Put Copy Part to AZURE with large object', -function describeF() { +describeSkipIfNotMultiple('Put Copy Part to AZURE with large object', function describeF() { this.timeout(800000); withV4(sigCfg => { beforeEach(() => { @@ -659,54 +700,59 @@ function describeF() { afterEach(() => { process.stdout.write('Emptying bucket\n'); - return bucketUtil.empty(azureContainerName) - .then(() => { - process.stdout.write('Deleting bucket\n'); - return bucketUtil.deleteOne(azureContainerName); - }) - .catch(err => { - process.stdout.write(`Error in afterEach: ${err}\n`); - throw err; - }); + return bucketUtil + .empty(azureContainerName) + .then(() => { + process.stdout.write('Deleting bucket\n'); + return bucketUtil.deleteOne(azureContainerName); + }) + .catch(err => { + process.stdout.write(`Error in afterEach: ${err}\n`); + throw err; + }); }); describe('Basic test with large object: ', () => { beforeEach(function beF(done) { - this.currentTest.keyNameOneHundredAndFiveMbAzure = - `onehundredandfivembazure${uniqName(keyObjectAzure)}`; - this.currentTest.mpuKeyNameAzure = - `mpukeyname${uniqName(keyObjectAzure)}`; + this.currentTest.keyNameOneHundredAndFiveMbAzure = `onehundredandfivembazure${uniqName(keyObjectAzure)}`; + this.currentTest.mpuKeyNameAzure = `mpukeyname${uniqName(keyObjectAzure)}`; const params = { Bucket: azureContainerName, Key: this.currentTest.mpuKeyNameAzure, Metadata: { 'scal-location-constraint': azureLocation }, }; - async.waterfall([ - next => { - s3.send(new CreateBucketCommand({ Bucket: azureContainerName })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new PutObjectCommand({ - Bucket: azureContainerName, - Key: this.currentTest.keyNameOneHundredAndFiveMbAzure, - Body: oneHundredAndFiveMbBody, - Metadata: { 'scal-location-constraint': azureLocation }, - })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new CreateMultipartUploadCommand(params)) - .then(res => { - this.currentTest.uploadId = res.UploadId; - next(); - }) - .catch(err => next(new Error( - `createMultipartUpload: Expected success, got error: ${err}`))); - }, - ], done); + async.waterfall( + [ + next => { + s3.send(new CreateBucketCommand({ Bucket: azureContainerName })) + .then(() => next()) + .catch(next); + }, + next => { + s3.send( + new PutObjectCommand({ + Bucket: azureContainerName, + Key: this.currentTest.keyNameOneHundredAndFiveMbAzure, + Body: oneHundredAndFiveMbBody, + Metadata: { 'scal-location-constraint': azureLocation }, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + s3.send(new CreateMultipartUploadCommand(params)) + .then(res => { + this.currentTest.uploadId = res.UploadId; + next(); + }) + .catch(err => + next(new Error(`createMultipartUpload: Expected success, got error: ${err}`)), + ); + }, + ], + done, + ); }); afterEach(function afterEachF(done) { const params = { @@ -719,47 +765,43 @@ function describeF() { .catch(done); }); - it('should copy 105 MB part from Azure to MPU with Azure ' + - 'location', function ifF(done) { + it('should copy 105 MB part from Azure to MPU with Azure ' + 'location', function ifF(done) { const params = { Bucket: azureContainerName, - CopySource: - `${azureContainerName}/` + - `${this.test.keyNameOneHundredAndFiveMbAzure}`, + CopySource: `${azureContainerName}/` + `${this.test.keyNameOneHundredAndFiveMbAzure}`, Key: this.test.mpuKeyNameAzure, PartNumber: 1, UploadId: this.test.uploadId, }; - async.waterfall([ - next => { - s3.send(new UploadPartCopyCommand(params)) - .then(res => { - assert.strictEqual(res.ETag, - `"${oneHundredAndFiveMbMD5}"`); - next(); - }) - .catch(err => next(new Error( - `uploadPartCopy: Expected success, got error: ${err}`))); - }, - next => { - const infos = { - azureContainerName, - mpuKeyNameAzure: - this.test.mpuKeyNameAzure, - uploadId: this.test.uploadId, - md5: oneHundredAndFiveMbMD5, - subPartSize: [100 * 1024 * 1024, 5 * 1024 * 1024], - }; - assertCopyPart(infos, next); - }, - ], done); + async.waterfall( + [ + next => { + s3.send(new UploadPartCopyCommand(params)) + .then(res => { + assert.strictEqual(res.ETag, `"${oneHundredAndFiveMbMD5}"`); + next(); + }) + .catch(err => next(new Error(`uploadPartCopy: Expected success, got error: ${err}`))); + }, + next => { + const infos = { + azureContainerName, + mpuKeyNameAzure: this.test.mpuKeyNameAzure, + uploadId: this.test.uploadId, + md5: oneHundredAndFiveMbMD5, + subPartSize: [100 * 1024 * 1024, 5 * 1024 * 1024], + }; + assertCopyPart(infos, next); + }, + ], + done, + ); }); }); }); }); -describeSkipIfNotMultiple('Put Copy Part to AZURE with complete MPU', -function describeF() { +describeSkipIfNotMultiple('Put Copy Part to AZURE with complete MPU', function describeF() { this.timeout(800000); withV4(sigCfg => { beforeEach(() => { @@ -769,138 +811,138 @@ function describeF() { afterEach(() => { process.stdout.write('Emptying bucket azureContainerName\n'); - return bucketUtil.empty(azureContainerName) - .then(() => { - process.stdout.write('Deleting bucket azureContainerName\n'); - return bucketUtil.deleteOne(azureContainerName); - }) - .then(() => { - process.stdout.write('Emptying bucket awsBucketName\n'); - return bucketUtil.empty(awsBucketName); - }) - .then(() => { - process.stdout.write('Deleting bucket awsBucketName\n'); - return bucketUtil.deleteOne(awsBucketName); - }) - .catch(err => { - process.stdout.write(`Error in afterEach: ${err}\n`); - throw err; - }); + return bucketUtil + .empty(azureContainerName) + .then(() => { + process.stdout.write('Deleting bucket azureContainerName\n'); + return bucketUtil.deleteOne(azureContainerName); + }) + .then(() => { + process.stdout.write('Emptying bucket awsBucketName\n'); + return bucketUtil.empty(awsBucketName); + }) + .then(() => { + process.stdout.write('Deleting bucket awsBucketName\n'); + return bucketUtil.deleteOne(awsBucketName); + }) + .catch(err => { + process.stdout.write(`Error in afterEach: ${err}\n`); + throw err; + }); }); - describe('Basic test with complete MPU from AWS to Azure location: ', - () => { + describe('Basic test with complete MPU from AWS to Azure location: ', () => { beforeEach(function beF(done) { - this.currentTest.keyNameAws = - `onehundredandfivembazure${uniqName(keyObjectAWS)}`; - this.currentTest.mpuKeyNameAzure = - `mpukeyname${uniqName(keyObjectAzure)}`; + this.currentTest.keyNameAws = `onehundredandfivembazure${uniqName(keyObjectAWS)}`; + this.currentTest.mpuKeyNameAzure = `mpukeyname${uniqName(keyObjectAzure)}`; const createMpuParams = { Bucket: azureContainerName, Key: this.currentTest.mpuKeyNameAzure, Metadata: { 'scal-location-constraint': azureLocation }, }; - async.waterfall([ - next => { - s3.send(new CreateBucketCommand({ Bucket: awsBucketName })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new CreateBucketCommand({ Bucket: azureContainerName })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new PutObjectCommand({ - Bucket: awsBucketName, - Key: this.currentTest.keyNameAws, - Body: fiveMbBody, - Metadata: { 'scal-location-constraint': awsLocation }, - })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new CreateMultipartUploadCommand(createMpuParams)) - .then(res => { - this.currentTest.uploadId = res.UploadId; - next(); - }) - .catch(err => next(new Error( - `createMultipartUpload: Expected success, got error: ${err}`))); - }, - ], done); + async.waterfall( + [ + next => { + s3.send(new CreateBucketCommand({ Bucket: awsBucketName })) + .then(() => next()) + .catch(next); + }, + next => { + s3.send(new CreateBucketCommand({ Bucket: azureContainerName })) + .then(() => next()) + .catch(next); + }, + next => { + s3.send( + new PutObjectCommand({ + Bucket: awsBucketName, + Key: this.currentTest.keyNameAws, + Body: fiveMbBody, + Metadata: { 'scal-location-constraint': awsLocation }, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + s3.send(new CreateMultipartUploadCommand(createMpuParams)) + .then(res => { + this.currentTest.uploadId = res.UploadId; + next(); + }) + .catch(err => + next(new Error(`createMultipartUpload: Expected success, got error: ${err}`)), + ); + }, + ], + done, + ); }); - it('should copy two 5 MB part from Azure to MPU with Azure ' + - 'location', function ifF(done) { + it('should copy two 5 MB part from Azure to MPU with Azure ' + 'location', function ifF(done) { const uploadParams = { Bucket: azureContainerName, - CopySource: - `${awsBucketName}/` + - `${this.test.keyNameAws}`, + CopySource: `${awsBucketName}/` + `${this.test.keyNameAws}`, Key: this.test.mpuKeyNameAzure, PartNumber: 1, UploadId: this.test.uploadId, }; const uploadParams2 = { Bucket: azureContainerName, - CopySource: - `${awsBucketName}/` + - `${this.test.keyNameAws}`, + CopySource: `${awsBucketName}/` + `${this.test.keyNameAws}`, Key: this.test.mpuKeyNameAzure, PartNumber: 2, UploadId: this.test.uploadId, }; - async.waterfall([ - next => { - s3.send(new UploadPartCopyCommand(uploadParams)) - .then(res => { - assert.strictEqual(res.ETag, `"${fiveMbMD5}"`); - next(); - }) - .catch(err => next(new Error( - `uploadPartCopy: Expected success, got error: ${err}`))); - }, - next => { - s3.send(new UploadPartCopyCommand(uploadParams2)) - .then(res => { - assert.strictEqual(res.ETag, `"${fiveMbMD5}"`); - next(); - }) - .catch(err => next(new Error( - `uploadPartCopy: Expected success, got error: ${err}`))); - }, - next => { - const completeMpuParams = { - Bucket: azureContainerName, - Key: this.test.mpuKeyNameAzure, - MultipartUpload: { - Parts: [ - { - ETag: `"${fiveMbMD5}"`, - PartNumber: 1, - }, - { - ETag: `"${fiveMbMD5}"`, - PartNumber: 2, - }, - ], - }, - UploadId: this.test.uploadId, - }; - s3.send(new CompleteMultipartUploadCommand(completeMpuParams)) - .then(res => { - assert.strictEqual(res.Bucket, azureContainerName); - assert.strictEqual(res.Key, - this.test.mpuKeyNameAzure); - next(); - }) - .catch(err => next(new Error( - `completeMultipartUpload: Expected success, got error: ${err}`))); - }, - ], done); + async.waterfall( + [ + next => { + s3.send(new UploadPartCopyCommand(uploadParams)) + .then(res => { + assert.strictEqual(res.ETag, `"${fiveMbMD5}"`); + next(); + }) + .catch(err => next(new Error(`uploadPartCopy: Expected success, got error: ${err}`))); + }, + next => { + s3.send(new UploadPartCopyCommand(uploadParams2)) + .then(res => { + assert.strictEqual(res.ETag, `"${fiveMbMD5}"`); + next(); + }) + .catch(err => next(new Error(`uploadPartCopy: Expected success, got error: ${err}`))); + }, + next => { + const completeMpuParams = { + Bucket: azureContainerName, + Key: this.test.mpuKeyNameAzure, + MultipartUpload: { + Parts: [ + { + ETag: `"${fiveMbMD5}"`, + PartNumber: 1, + }, + { + ETag: `"${fiveMbMD5}"`, + PartNumber: 2, + }, + ], + }, + UploadId: this.test.uploadId, + }; + s3.send(new CompleteMultipartUploadCommand(completeMpuParams)) + .then(res => { + assert.strictEqual(res.Bucket, azureContainerName); + assert.strictEqual(res.Key, this.test.mpuKeyNameAzure); + next(); + }) + .catch(err => + next(new Error(`completeMultipartUpload: Expected success, got error: ${err}`)), + ); + }, + ], + done, + ); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/objectPutCopyPart/objectPutCopyPartGcp.js b/tests/functional/aws-node-sdk/test/multipleBackend/objectPutCopyPart/objectPutCopyPartGcp.js index b6e416ab4b..3d99f9a9f4 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/objectPutCopyPart/objectPutCopyPartGcp.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/objectPutCopyPart/objectPutCopyPartGcp.js @@ -15,9 +15,19 @@ const { const { config } = require('../../../../../../lib/Config'); const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); -const { uniqName, gcpBucketMPU, - gcpClient, gcpLocation, gcpLocationMismatch, memLocation, - awsLocation, awsS3, getOwnerInfo, genUniqID, describeSkipIfNotMultiple } = require('../utils'); +const { + uniqName, + gcpBucketMPU, + gcpClient, + gcpLocation, + gcpLocationMismatch, + memLocation, + awsLocation, + awsS3, + getOwnerInfo, + genUniqID, + describeSkipIfNotMultiple, +} = require('../utils'); const bucket = `partcopygcp${genUniqID()}`; @@ -51,12 +61,8 @@ const result = { MaxParts: 1000, IsTruncated: false, Parts: [], - Initiator: - { ID: ownerID, - DisplayName: ownerDisplayName }, - Owner: - { DisplayName: ownerDisplayName, - ID: ownerID }, + Initiator: { ID: ownerID, DisplayName: ownerDisplayName }, + Owner: { DisplayName: ownerDisplayName, ID: ownerID }, StorageClass: 'STANDARD', }; @@ -69,36 +75,46 @@ function assertCopyPart(infos, cb) { resultCopy.Bucket = bucketName; resultCopy.Key = keyName; resultCopy.UploadId = uploadId; - async.waterfall([ - next => { - s3.send(new ListPartsCommand({ - Bucket: bucketName, - Key: keyName, - UploadId: uploadId, - })) - .then(res => { - resultCopy.Parts = - [{ PartNumber: 1, - LastModified: res.Parts[0].LastModified, - ETag: `"${md5}"`, - Size: totalSize }]; - assert.deepStrictEqual(res, resultCopy); - next(); - }) - .catch(err => next(new Error( - `listParts: Expected success, got error: ${err}`))); - }, - next => gcpClient.listParts({ - Bucket: gcpBucketMPU, - Key: keyName, - UploadId: uploadId, - }, (err, res) => { - assert.ifError(err, 'GCP listParts: Expected success,' + - `got error: ${err}`); - assert.strictEqual(res.Contents[0].ETag, `"${md5}"`); - next(); - }), - ], cb); + async.waterfall( + [ + next => { + s3.send( + new ListPartsCommand({ + Bucket: bucketName, + Key: keyName, + UploadId: uploadId, + }), + ) + .then(res => { + resultCopy.Parts = [ + { + PartNumber: 1, + LastModified: res.Parts[0].LastModified, + ETag: `"${md5}"`, + Size: totalSize, + }, + ]; + assert.deepStrictEqual(res, resultCopy); + next(); + }) + .catch(err => next(new Error(`listParts: Expected success, got error: ${err}`))); + }, + next => + gcpClient.listParts( + { + Bucket: gcpBucketMPU, + Key: keyName, + UploadId: uploadId, + }, + (err, res) => { + assert.ifError(err, 'GCP listParts: Expected success,' + `got error: ${err}`); + assert.strictEqual(res.Contents[0].ETag, `"${md5}"`); + next(); + }, + ), + ], + cb, + ); } describeSkipIfNotMultiple('Put Copy Part to GCP', function describeFn() { @@ -107,52 +123,48 @@ describeSkipIfNotMultiple('Put Copy Part to GCP', function describeFn() { beforeEach(done => { bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; - s3.send(new CreateBucketCommand({ - Bucket: bucket, - CreateBucketConfiguration: { - LocationConstraint: gcpLocation, - }, - })) + s3.send( + new CreateBucketCommand({ + Bucket: bucket, + CreateBucketConfiguration: { + LocationConstraint: gcpLocation, + }, + }), + ) .then(() => done()) .catch(done); }); afterEach(() => { process.stdout.write('Emptying bucket\n'); - return bucketUtil.empty(bucket) - .then(() => bucketUtil.empty(memBucketName)) - .then(() => { - process.stdout.write(`Deleting bucket ${bucket}\n`); - return bucketUtil.deleteOne(bucket); - }) - .then(() => { - process.stdout.write(`Deleting bucket ${memBucketName}\n`); - return bucketUtil.deleteOne(memBucketName); - }) - .catch(err => { - process.stdout.write(`Error in afterEach: ${err}\n`); - throw err; - }); + return bucketUtil + .empty(bucket) + .then(() => bucketUtil.empty(memBucketName)) + .then(() => { + process.stdout.write(`Deleting bucket ${bucket}\n`); + return bucketUtil.deleteOne(bucket); + }) + .then(() => { + process.stdout.write(`Deleting bucket ${memBucketName}\n`); + return bucketUtil.deleteOne(memBucketName); + }) + .catch(err => { + process.stdout.write(`Error in afterEach: ${err}\n`); + throw err; + }); }); describe('Basic test: ', () => { beforeEach(function beforeFn(done) { - this.currentTest.keyNameNormalGcp = - `normalgcp${uniqName(keyObjectGcp)}`; - this.currentTest.keyNameNormalGcpMismatch = - `normalgcpmismatch${uniqName(keyObjectGcp)}`; + this.currentTest.keyNameNormalGcp = `normalgcp${uniqName(keyObjectGcp)}`; + this.currentTest.keyNameNormalGcpMismatch = `normalgcpmismatch${uniqName(keyObjectGcp)}`; - this.currentTest.keyNameFiveMbGcp = - `fivembgcp${uniqName(keyObjectGcp)}`; - this.currentTest.keyNameFiveMbMem = - `fivembmem${uniqName(keyObjectMemory)}`; + this.currentTest.keyNameFiveMbGcp = `fivembgcp${uniqName(keyObjectGcp)}`; + this.currentTest.keyNameFiveMbMem = `fivembmem${uniqName(keyObjectMemory)}`; - this.currentTest.mpuKeyNameGcp = - `mpukeyname${uniqName(keyObjectGcp)}`; - this.currentTest.mpuKeyNameMem = - `mpukeyname${uniqName(keyObjectMemory)}`; - this.currentTest.mpuKeyNameAWS = - `mpukeyname${uniqName(keyObjectAWS)}`; + this.currentTest.mpuKeyNameGcp = `mpukeyname${uniqName(keyObjectGcp)}`; + this.currentTest.mpuKeyNameMem = `mpukeyname${uniqName(keyObjectMemory)}`; + this.currentTest.mpuKeyNameAWS = `mpukeyname${uniqName(keyObjectAWS)}`; const paramsGcp = { Bucket: bucket, Key: this.currentTest.mpuKeyNameGcp, @@ -168,86 +180,107 @@ describeSkipIfNotMultiple('Put Copy Part to GCP', function describeFn() { Key: this.currentTest.mpuKeyNameAWS, Metadata: { 'scal-location-constraint': awsLocation }, }; - async.waterfall([ - next => { - s3.send(new CreateBucketCommand({ Bucket: bucket })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new CreateBucketCommand({ Bucket: memBucketName })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: this.currentTest.keyNameNormalGcp, - Body: normalBody, - Metadata: { 'scal-location-constraint': gcpLocation }, - })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: this.currentTest.keyNameNormalGcpMismatch, - Body: normalBody, - Metadata: { 'scal-location-constraint': - gcpLocationMismatch }, - })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: this.currentTest.keyNameFiveMbGcp, - Body: fiveMbBody, - Metadata: { 'scal-location-constraint': gcpLocation }, - })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: this.currentTest.keyNameFiveMbMem, - Body: fiveMbBody, - Metadata: { 'scal-location-constraint': memLocation }, - })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new CreateMultipartUploadCommand(paramsGcp)) - .then(res => { - this.currentTest.uploadId = res.UploadId; - next(); - }) - .catch(err => next(new Error( - `createMultipartUpload on gcp: Expected success, got error: ${err}`))); - }, - next => { - s3.send(new CreateMultipartUploadCommand(paramsMem)) - .then(res => { - this.currentTest.uploadIdMem = res.UploadId; - next(); - }) - .catch(err => next(new Error( - `createMultipartUpload in memory: Expected success, got error: ${err}`))); - }, - next => { - s3.send(new CreateMultipartUploadCommand(paramsAWS)) - .then(res => { - this.currentTest.uploadIdAWS = res.UploadId; - next(); - }) - .catch(err => next(new Error( - `createMultipartUpload on AWS: Expected success, got error: ${err}`))); - }, - ], done); + async.waterfall( + [ + next => { + s3.send(new CreateBucketCommand({ Bucket: bucket })) + .then(() => next()) + .catch(next); + }, + next => { + s3.send(new CreateBucketCommand({ Bucket: memBucketName })) + .then(() => next()) + .catch(next); + }, + next => { + s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: this.currentTest.keyNameNormalGcp, + Body: normalBody, + Metadata: { 'scal-location-constraint': gcpLocation }, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: this.currentTest.keyNameNormalGcpMismatch, + Body: normalBody, + Metadata: { 'scal-location-constraint': gcpLocationMismatch }, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: this.currentTest.keyNameFiveMbGcp, + Body: fiveMbBody, + Metadata: { 'scal-location-constraint': gcpLocation }, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: this.currentTest.keyNameFiveMbMem, + Body: fiveMbBody, + Metadata: { 'scal-location-constraint': memLocation }, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + s3.send(new CreateMultipartUploadCommand(paramsGcp)) + .then(res => { + this.currentTest.uploadId = res.UploadId; + next(); + }) + .catch(err => + next( + new Error(`createMultipartUpload on gcp: Expected success, got error: ${err}`), + ), + ); + }, + next => { + s3.send(new CreateMultipartUploadCommand(paramsMem)) + .then(res => { + this.currentTest.uploadIdMem = res.UploadId; + next(); + }) + .catch(err => + next( + new Error( + `createMultipartUpload in memory: Expected success, got error: ${err}`, + ), + ), + ); + }, + next => { + s3.send(new CreateMultipartUploadCommand(paramsAWS)) + .then(res => { + this.currentTest.uploadIdAWS = res.UploadId; + next(); + }) + .catch(err => + next( + new Error(`createMultipartUpload on AWS: Expected success, got error: ${err}`), + ), + ); + }, + ], + done, + ); }); afterEach(function afterFn(done) { @@ -266,301 +299,301 @@ describeSkipIfNotMultiple('Put Copy Part to GCP', function describeFn() { Key: this.currentTest.mpuKeyNameAWS, UploadId: this.currentTest.uploadIdAWS, }; - async.waterfall([ - next => { - s3.send(new AbortMultipartUploadCommand(paramsGcp)) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new AbortMultipartUploadCommand(paramsMem)) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new AbortMultipartUploadCommand(paramsAWS)) - .then(() => next()) - .catch(next); - }, - ], done); + async.waterfall( + [ + next => { + s3.send(new AbortMultipartUploadCommand(paramsGcp)) + .then(() => next()) + .catch(next); + }, + next => { + s3.send(new AbortMultipartUploadCommand(paramsMem)) + .then(() => next()) + .catch(next); + }, + next => { + s3.send(new AbortMultipartUploadCommand(paramsAWS)) + .then(() => next()) + .catch(next); + }, + ], + done, + ); }); - it('should copy small part from GCP to MPU with GCP location', - function itFn(done) { + it('should copy small part from GCP to MPU with GCP location', function itFn(done) { const params = { Bucket: bucket, - CopySource: - `${bucket}/${this.test.keyNameNormalGcp}`, + CopySource: `${bucket}/${this.test.keyNameNormalGcp}`, Key: this.test.mpuKeyNameGcp, PartNumber: 1, UploadId: this.test.uploadId, }; - async.waterfall([ - next => { - s3.send(new UploadPartCopyCommand(params)) - .then(res => { - assert.strictEqual(res.ETag, `"${normalMD5}"`); - next(); - }) - .catch(err => next(new Error( - `uploadPartCopy: Expected success, got error: ${err}`))); - }, - next => { - const infos = { - bucketName: bucket, - keyName: this.test.mpuKeyNameGcp, - uploadId: this.test.uploadId, - md5: normalMD5, - totalSize: normalBodySize, - }; - assertCopyPart(infos, next); - }, - ], done); + async.waterfall( + [ + next => { + s3.send(new UploadPartCopyCommand(params)) + .then(res => { + assert.strictEqual(res.ETag, `"${normalMD5}"`); + next(); + }) + .catch(err => next(new Error(`uploadPartCopy: Expected success, got error: ${err}`))); + }, + next => { + const infos = { + bucketName: bucket, + keyName: this.test.mpuKeyNameGcp, + uploadId: this.test.uploadId, + md5: normalMD5, + totalSize: normalBodySize, + }; + assertCopyPart(infos, next); + }, + ], + done, + ); }); - it('should copy small part from GCP with bucketMatch=false to ' + - 'MPU with GCP location', - function itFn(done) { - const params = { - Bucket: bucket, - CopySource: - `${bucket}/${this.test.keyNameNormalGcpMismatch}`, - Key: this.test.mpuKeyNameGcp, - PartNumber: 1, - UploadId: this.test.uploadId, - }; - async.waterfall([ - next => { - s3.send(new UploadPartCopyCommand(params)) - .then(res => { - assert.strictEqual(res.ETag, `"${normalMD5}"`); - next(); - }) - .catch(err => next(new Error( - `uploadPartCopy: Expected success, got error: ${err}`))); - }, - next => { - const infos = { - bucketName: bucket, - keyName: this.test.mpuKeyNameGcp, - uploadId: this.test.uploadId, - md5: normalMD5, - totalSize: normalBodySize, - }; - assertCopyPart(infos, next); - }, - ], done); - }); + it( + 'should copy small part from GCP with bucketMatch=false to ' + 'MPU with GCP location', + function itFn(done) { + const params = { + Bucket: bucket, + CopySource: `${bucket}/${this.test.keyNameNormalGcpMismatch}`, + Key: this.test.mpuKeyNameGcp, + PartNumber: 1, + UploadId: this.test.uploadId, + }; + async.waterfall( + [ + next => { + s3.send(new UploadPartCopyCommand(params)) + .then(res => { + assert.strictEqual(res.ETag, `"${normalMD5}"`); + next(); + }) + .catch(err => + next(new Error(`uploadPartCopy: Expected success, got error: ${err}`)), + ); + }, + next => { + const infos = { + bucketName: bucket, + keyName: this.test.mpuKeyNameGcp, + uploadId: this.test.uploadId, + md5: normalMD5, + totalSize: normalBodySize, + }; + assertCopyPart(infos, next); + }, + ], + done, + ); + }, + ); - it('should copy 5 Mb part from GCP to MPU with GCP location', - function ifF(done) { + it('should copy 5 Mb part from GCP to MPU with GCP location', function ifF(done) { const params = { Bucket: bucket, - CopySource: - `${bucket}/${this.test.keyNameFiveMbGcp}`, + CopySource: `${bucket}/${this.test.keyNameFiveMbGcp}`, Key: this.test.mpuKeyNameGcp, PartNumber: 1, UploadId: this.test.uploadId, }; - async.waterfall([ - next => { - s3.send(new UploadPartCopyCommand(params)) - .then(res => { - assert.strictEqual(res.ETag, `"${fiveMbMD5}"`); - next(); - }) - .catch(err => next(new Error( - `uploadPartCopy: Expected success, got error: ${err}`))); - }, - next => { - const infos = { - bucketName: bucket, - keyName: this.test.mpuKeyNameGcp, - uploadId: this.test.uploadId, - md5: fiveMbMD5, - totalSize: fiveMB, - }; - assertCopyPart(infos, next); - }, - ], done); + async.waterfall( + [ + next => { + s3.send(new UploadPartCopyCommand(params)) + .then(res => { + assert.strictEqual(res.ETag, `"${fiveMbMD5}"`); + next(); + }) + .catch(err => next(new Error(`uploadPartCopy: Expected success, got error: ${err}`))); + }, + next => { + const infos = { + bucketName: bucket, + keyName: this.test.mpuKeyNameGcp, + uploadId: this.test.uploadId, + md5: fiveMbMD5, + totalSize: fiveMB, + }; + assertCopyPart(infos, next); + }, + ], + done, + ); }); - it('should copy part from GCP to MPU with memory location', - function ifF(done) { + it('should copy part from GCP to MPU with memory location', function ifF(done) { const params = { Bucket: memBucketName, - CopySource: - `${bucket}/${this.test.keyNameNormalGcp}`, + CopySource: `${bucket}/${this.test.keyNameNormalGcp}`, Key: this.test.mpuKeyNameMem, PartNumber: 1, UploadId: this.test.uploadIdMem, }; - async.waterfall([ - next => { - s3.send(new UploadPartCopyCommand(params)) - .then(res => { - assert.strictEqual(res.ETag, `"${normalMD5}"`); - next(); - }) - .catch(err => next(new Error( - `uploadPartCopy: Expected success, got error: ${err}`))); - }, - next => { - s3.send(new ListPartsCommand({ - Bucket: memBucketName, - Key: this.test.mpuKeyNameMem, - UploadId: this.test.uploadIdMem, - })) - .then(res => { - const resultCopy = - JSON.parse(JSON.stringify(result)); - resultCopy.Bucket = memBucketName; - resultCopy.Key = this.test.mpuKeyNameMem; - resultCopy.UploadId = this.test.uploadIdMem; - resultCopy.Parts = - [{ PartNumber: 1, - LastModified: res.Parts[0].LastModified, - ETag: `"${normalMD5}"`, - Size: normalBodySize }]; - assert.deepStrictEqual(res, resultCopy); - next(); - }) - .catch(err => next(new Error( - `listParts: Expected success, got error: ${err}`))); - }, - ], done); + async.waterfall( + [ + next => { + s3.send(new UploadPartCopyCommand(params)) + .then(res => { + assert.strictEqual(res.ETag, `"${normalMD5}"`); + next(); + }) + .catch(err => next(new Error(`uploadPartCopy: Expected success, got error: ${err}`))); + }, + next => { + s3.send( + new ListPartsCommand({ + Bucket: memBucketName, + Key: this.test.mpuKeyNameMem, + UploadId: this.test.uploadIdMem, + }), + ) + .then(res => { + const resultCopy = JSON.parse(JSON.stringify(result)); + resultCopy.Bucket = memBucketName; + resultCopy.Key = this.test.mpuKeyNameMem; + resultCopy.UploadId = this.test.uploadIdMem; + resultCopy.Parts = [ + { + PartNumber: 1, + LastModified: res.Parts[0].LastModified, + ETag: `"${normalMD5}"`, + Size: normalBodySize, + }, + ]; + assert.deepStrictEqual(res, resultCopy); + next(); + }) + .catch(err => next(new Error(`listParts: Expected success, got error: ${err}`))); + }, + ], + done, + ); }); - it('should copy part from GCP to MPU with AWS location', - function ifF(done) { + it('should copy part from GCP to MPU with AWS location', function ifF(done) { const params = { Bucket: memBucketName, - CopySource: - `${bucket}/${this.test.keyNameNormalGcp}`, + CopySource: `${bucket}/${this.test.keyNameNormalGcp}`, Key: this.test.mpuKeyNameAWS, PartNumber: 1, UploadId: this.test.uploadIdAWS, }; - async.waterfall([ - next => { - s3.send(new UploadPartCopyCommand(params)) - .then(res => { - assert.strictEqual(res.ETag, `"${normalMD5}"`); - next(); - }) - .catch(err => next(new Error( - `uploadPartCopy: Expected success, got error: ${err}`))); - }, - next => { - const awsBucket = - config.locationConstraints[awsLocation] - .details.bucketName; - awsS3.listParts({ - Bucket: awsBucket, - Key: this.test.mpuKeyNameAWS, - UploadId: this.test.uploadIdAWS, - }, (err, res) => { - assert.ifError(err, - 'listParts: Expected success,' + - ` got error: ${err}`); - assert.strictEqual(res.Bucket, awsBucket); - assert.strictEqual(res.Key, - this.test.mpuKeyNameAWS); - assert.strictEqual(res.UploadId, - this.test.uploadIdAWS); - assert.strictEqual(res.Parts.length, 1); - assert.strictEqual(res.Parts[0].PartNumber, 1); - assert.strictEqual(res.Parts[0].ETag, - `"${normalMD5}"`); - assert.strictEqual(res.Parts[0].Size, - normalBodySize); - next(); - }); - }, - ], done); + async.waterfall( + [ + next => { + s3.send(new UploadPartCopyCommand(params)) + .then(res => { + assert.strictEqual(res.ETag, `"${normalMD5}"`); + next(); + }) + .catch(err => next(new Error(`uploadPartCopy: Expected success, got error: ${err}`))); + }, + next => { + const awsBucket = config.locationConstraints[awsLocation].details.bucketName; + awsS3.listParts( + { + Bucket: awsBucket, + Key: this.test.mpuKeyNameAWS, + UploadId: this.test.uploadIdAWS, + }, + (err, res) => { + assert.ifError(err, 'listParts: Expected success,' + ` got error: ${err}`); + assert.strictEqual(res.Bucket, awsBucket); + assert.strictEqual(res.Key, this.test.mpuKeyNameAWS); + assert.strictEqual(res.UploadId, this.test.uploadIdAWS); + assert.strictEqual(res.Parts.length, 1); + assert.strictEqual(res.Parts[0].PartNumber, 1); + assert.strictEqual(res.Parts[0].ETag, `"${normalMD5}"`); + assert.strictEqual(res.Parts[0].Size, normalBodySize); + next(); + }, + ); + }, + ], + done, + ); }); - it('should copy part from GCP object with range to MPU ' + - 'with AWS location', function ifF(done) { + it('should copy part from GCP object with range to MPU ' + 'with AWS location', function ifF(done) { const params = { Bucket: memBucketName, - CopySource: - `${bucket}/${this.test.keyNameNormalGcp}`, + CopySource: `${bucket}/${this.test.keyNameNormalGcp}`, Key: this.test.mpuKeyNameAWS, CopySourceRange: 'bytes=0-5', PartNumber: 1, UploadId: this.test.uploadIdAWS, }; - async.waterfall([ - next => { - s3.send(new UploadPartCopyCommand(params)) - .then(res => { - assert.strictEqual(res.ETag, `"${sixBytesMD5}"`); - next(); - }) - .catch(err => next(new Error( - `uploadPartCopy: Expected success, got error: ${err}`))); - }, - next => { - const awsBucket = - config.locationConstraints[awsLocation] - .details.bucketName; - awsS3.listParts({ - Bucket: awsBucket, - Key: this.test.mpuKeyNameAWS, - UploadId: this.test.uploadIdAWS, - }, (err, res) => { - assert.ifError(err, - 'listParts: Expected success,' + - ` got error: ${err}`); - assert.strictEqual(res.Bucket, awsBucket); - assert.strictEqual(res.Key, - this.test.mpuKeyNameAWS); - assert.strictEqual(res.UploadId, - this.test.uploadIdAWS); - assert.strictEqual(res.Parts.length, 1); - assert.strictEqual(res.Parts[0].PartNumber, 1); - assert.strictEqual(res.Parts[0].ETag, - `"${sixBytesMD5}"`); - assert.strictEqual(res.Parts[0].Size, 6); - next(); - }); - }, - ], done); + async.waterfall( + [ + next => { + s3.send(new UploadPartCopyCommand(params)) + .then(res => { + assert.strictEqual(res.ETag, `"${sixBytesMD5}"`); + next(); + }) + .catch(err => next(new Error(`uploadPartCopy: Expected success, got error: ${err}`))); + }, + next => { + const awsBucket = config.locationConstraints[awsLocation].details.bucketName; + awsS3.listParts( + { + Bucket: awsBucket, + Key: this.test.mpuKeyNameAWS, + UploadId: this.test.uploadIdAWS, + }, + (err, res) => { + assert.ifError(err, 'listParts: Expected success,' + ` got error: ${err}`); + assert.strictEqual(res.Bucket, awsBucket); + assert.strictEqual(res.Key, this.test.mpuKeyNameAWS); + assert.strictEqual(res.UploadId, this.test.uploadIdAWS); + assert.strictEqual(res.Parts.length, 1); + assert.strictEqual(res.Parts[0].PartNumber, 1); + assert.strictEqual(res.Parts[0].ETag, `"${sixBytesMD5}"`); + assert.strictEqual(res.Parts[0].Size, 6); + next(); + }, + ); + }, + ], + done, + ); }); - it('should copy 5 Mb part from a memory location to MPU with ' + - 'GCP location', - function ifF(done) { + it('should copy 5 Mb part from a memory location to MPU with ' + 'GCP location', function ifF(done) { const params = { Bucket: bucket, - CopySource: - `${bucket}/${this.test.keyNameFiveMbMem}`, + CopySource: `${bucket}/${this.test.keyNameFiveMbMem}`, Key: this.test.mpuKeyNameGcp, PartNumber: 1, UploadId: this.test.uploadId, }; - async.waterfall([ - next => { - s3.send(new UploadPartCopyCommand(params)) - .then(res => { - assert.strictEqual(res.ETag, `"${fiveMbMD5}"`); - next(); - }) - .catch(err => next(new Error( - `uploadPartCopy: Expected success, got error: ${err}`))); - }, - next => { - const infos = { - bucketName: bucket, - keyName: this.test.mpuKeyNameGcp, - uploadId: this.test.uploadId, - md5: fiveMbMD5, - totalSize: fiveMB, - }; - assertCopyPart(infos, next); - }, - ], done); + async.waterfall( + [ + next => { + s3.send(new UploadPartCopyCommand(params)) + .then(res => { + assert.strictEqual(res.ETag, `"${fiveMbMD5}"`); + next(); + }) + .catch(err => next(new Error(`uploadPartCopy: Expected success, got error: ${err}`))); + }, + next => { + const infos = { + bucketName: bucket, + keyName: this.test.mpuKeyNameGcp, + uploadId: this.test.uploadId, + md5: fiveMbMD5, + totalSize: fiveMB, + }; + assertCopyPart(infos, next); + }, + ], + done, + ); }); describe('with existing part', () => { @@ -576,75 +609,82 @@ describeSkipIfNotMultiple('Put Copy Part to GCP', function describeFn() { .then(() => done()) .catch(done); }); - it('should copy part from GCP to GCP with existing ' + - 'parts', function ifF(done) { + it('should copy part from GCP to GCP with existing ' + 'parts', function ifF(done) { const resultCopy = JSON.parse(JSON.stringify(result)); const params = { Bucket: bucket, - CopySource: - `${bucket}/${this.test.keyNameNormalGcp}`, + CopySource: `${bucket}/${this.test.keyNameNormalGcp}`, Key: this.test.mpuKeyNameGcp, PartNumber: 2, UploadId: this.test.uploadId, }; - async.waterfall([ - next => { - s3.send(new UploadPartCopyCommand(params)) - .then(res => { - assert.strictEqual(res.ETag, `"${normalMD5}"`); - next(); - }) - .catch(err => next(new Error( - `uploadPartCopy: Expected success, got error: ${err}`))); - }, - next => { - s3.send(new ListPartsCommand({ - Bucket: bucket, - Key: this.test.mpuKeyNameGcp, - UploadId: this.test.uploadId, - })) - .then(res => { - resultCopy.Bucket = bucket; - resultCopy.Key = this.test.mpuKeyNameGcp; - resultCopy.UploadId = this.test.uploadId; - resultCopy.Parts = - [{ PartNumber: 1, - LastModified: res.Parts[0].LastModified, - ETag: `"${oneKbMD5}"`, - Size: oneKb }, - { PartNumber: 2, - LastModified: res.Parts[1].LastModified, - ETag: `"${normalMD5}"`, - Size: 11 }, - ]; - assert.deepStrictEqual(res, resultCopy); - next(); - }) - .catch(err => next(new Error( - `listParts: Expected success, got error: ${err}`))); - }, - next => gcpClient.listParts({ - Bucket: gcpBucketMPU, - Key: this.test.mpuKeyNameGcp, - UploadId: this.test.uploadId, - }, (err, res) => { - assert.ifError(err, 'GCP listParts: Expected ' + - `success, got error: ${err}`); - assert.strictEqual( - res.Contents[0].ETag, `"${oneKbMD5}"`); - assert.strictEqual( - res.Contents[1].ETag, `"${normalMD5}"`); - next(); - }), - ], done); + async.waterfall( + [ + next => { + s3.send(new UploadPartCopyCommand(params)) + .then(res => { + assert.strictEqual(res.ETag, `"${normalMD5}"`); + next(); + }) + .catch(err => + next(new Error(`uploadPartCopy: Expected success, got error: ${err}`)), + ); + }, + next => { + s3.send( + new ListPartsCommand({ + Bucket: bucket, + Key: this.test.mpuKeyNameGcp, + UploadId: this.test.uploadId, + }), + ) + .then(res => { + resultCopy.Bucket = bucket; + resultCopy.Key = this.test.mpuKeyNameGcp; + resultCopy.UploadId = this.test.uploadId; + resultCopy.Parts = [ + { + PartNumber: 1, + LastModified: res.Parts[0].LastModified, + ETag: `"${oneKbMD5}"`, + Size: oneKb, + }, + { + PartNumber: 2, + LastModified: res.Parts[1].LastModified, + ETag: `"${normalMD5}"`, + Size: 11, + }, + ]; + assert.deepStrictEqual(res, resultCopy); + next(); + }) + .catch(err => next(new Error(`listParts: Expected success, got error: ${err}`))); + }, + next => + gcpClient.listParts( + { + Bucket: gcpBucketMPU, + Key: this.test.mpuKeyNameGcp, + UploadId: this.test.uploadId, + }, + (err, res) => { + assert.ifError(err, 'GCP listParts: Expected ' + `success, got error: ${err}`); + assert.strictEqual(res.Contents[0].ETag, `"${oneKbMD5}"`); + assert.strictEqual(res.Contents[1].ETag, `"${normalMD5}"`); + next(); + }, + ), + ], + done, + ); }); }); }); }); }); -describeSkipIfNotMultiple('Put Copy Part to GCP with complete MPU', -function describeF() { +describeSkipIfNotMultiple('Put Copy Part to GCP with complete MPU', function describeF() { this.timeout(800000); withV4(sigCfg => { beforeEach(() => { @@ -654,138 +694,138 @@ function describeF() { afterEach(() => { process.stdout.write('Emptying bucket\n'); - return bucketUtil.empty(bucket) - .then(() => { - process.stdout.write('Deleting bucket\n'); - return bucketUtil.deleteOne(bucket); - }) - .then(() => { - process.stdout.write('Emptying bucket awsBucketName\n'); - return bucketUtil.empty(awsBucketName); - }) - .then(() => { - process.stdout.write('Deleting bucket awsBucketName\n'); - return bucketUtil.deleteOne(awsBucketName); - }) - .catch(err => { - process.stdout.write(`Error in afterEach: ${err}\n`); - throw err; - }); + return bucketUtil + .empty(bucket) + .then(() => { + process.stdout.write('Deleting bucket\n'); + return bucketUtil.deleteOne(bucket); + }) + .then(() => { + process.stdout.write('Emptying bucket awsBucketName\n'); + return bucketUtil.empty(awsBucketName); + }) + .then(() => { + process.stdout.write('Deleting bucket awsBucketName\n'); + return bucketUtil.deleteOne(awsBucketName); + }) + .catch(err => { + process.stdout.write(`Error in afterEach: ${err}\n`); + throw err; + }); }); - describe('Basic test with complete MPU from AWS to GCP location: ', - () => { + describe('Basic test with complete MPU from AWS to GCP location: ', () => { beforeEach(function beF(done) { - this.currentTest.keyNameAws = - `onehundredandfivembgcp${uniqName(keyObjectAWS)}`; - this.currentTest.mpuKeyNameGcp = - `mpukeyname${uniqName(keyObjectGcp)}`; + this.currentTest.keyNameAws = `onehundredandfivembgcp${uniqName(keyObjectAWS)}`; + this.currentTest.mpuKeyNameGcp = `mpukeyname${uniqName(keyObjectGcp)}`; const createMpuParams = { Bucket: bucket, Key: this.currentTest.mpuKeyNameGcp, Metadata: { 'scal-location-constraint': gcpLocation }, }; - async.waterfall([ - next => { - s3.send(new CreateBucketCommand({ Bucket: awsBucketName })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new CreateBucketCommand({ Bucket: bucket })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new PutObjectCommand({ - Bucket: awsBucketName, - Key: this.currentTest.keyNameAws, - Body: fiveMbBody, - Metadata: { 'scal-location-constraint': awsLocation }, - })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new CreateMultipartUploadCommand(createMpuParams)) - .then(res => { - this.currentTest.uploadId = res.UploadId; - next(); - }) - .catch(err => next(new Error( - `createMultipartUpload: Expected success, got error: ${err}`))); - }, - ], done); + async.waterfall( + [ + next => { + s3.send(new CreateBucketCommand({ Bucket: awsBucketName })) + .then(() => next()) + .catch(next); + }, + next => { + s3.send(new CreateBucketCommand({ Bucket: bucket })) + .then(() => next()) + .catch(next); + }, + next => { + s3.send( + new PutObjectCommand({ + Bucket: awsBucketName, + Key: this.currentTest.keyNameAws, + Body: fiveMbBody, + Metadata: { 'scal-location-constraint': awsLocation }, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + s3.send(new CreateMultipartUploadCommand(createMpuParams)) + .then(res => { + this.currentTest.uploadId = res.UploadId; + next(); + }) + .catch(err => + next(new Error(`createMultipartUpload: Expected success, got error: ${err}`)), + ); + }, + ], + done, + ); }); - it('should copy two 5 MB part from GCP to MPU with GCP' + - 'location', function ifF(done) { + it('should copy two 5 MB part from GCP to MPU with GCP' + 'location', function ifF(done) { const uploadParams = { Bucket: bucket, - CopySource: - `${awsBucketName}/` + - `${this.test.keyNameAws}`, + CopySource: `${awsBucketName}/` + `${this.test.keyNameAws}`, Key: this.test.mpuKeyNameGcp, PartNumber: 1, UploadId: this.test.uploadId, }; const uploadParams2 = { Bucket: bucket, - CopySource: - `${awsBucketName}/` + - `${this.test.keyNameAws}`, + CopySource: `${awsBucketName}/` + `${this.test.keyNameAws}`, Key: this.test.mpuKeyNameGcp, PartNumber: 2, UploadId: this.test.uploadId, }; - async.waterfall([ - next => { - s3.send(new UploadPartCopyCommand(uploadParams)) - .then(res => { - assert.strictEqual(res.ETag, `"${fiveMbMD5}"`); - next(); - }) - .catch(err => next(new Error( - `uploadPartCopy: Expected success, got error: ${err}`))); - }, - next => { - s3.send(new UploadPartCopyCommand(uploadParams2)) - .then(res => { - assert.strictEqual(res.ETag, `"${fiveMbMD5}"`); - next(); - }) - .catch(err => next(new Error( - `uploadPartCopy: Expected success, got error: ${err}`))); - }, - next => { - const completeMpuParams = { - Bucket: bucket, - Key: this.test.mpuKeyNameGcp, - MultipartUpload: { - Parts: [ - { - ETag: `"${fiveMbMD5}"`, - PartNumber: 1, - }, - { - ETag: `"${fiveMbMD5}"`, - PartNumber: 2, - }, - ], - }, - UploadId: this.test.uploadId, - }; - s3.send(new CompleteMultipartUploadCommand(completeMpuParams)) - .then(res => { - assert.strictEqual(res.Bucket, bucket); - assert.strictEqual(res.Key, - this.test.mpuKeyNameGcp); - next(); - }) - .catch(err => next(new Error( - `completeMultipartUpload: Expected success, got error: ${err}`))); - }, - ], done); + async.waterfall( + [ + next => { + s3.send(new UploadPartCopyCommand(uploadParams)) + .then(res => { + assert.strictEqual(res.ETag, `"${fiveMbMD5}"`); + next(); + }) + .catch(err => next(new Error(`uploadPartCopy: Expected success, got error: ${err}`))); + }, + next => { + s3.send(new UploadPartCopyCommand(uploadParams2)) + .then(res => { + assert.strictEqual(res.ETag, `"${fiveMbMD5}"`); + next(); + }) + .catch(err => next(new Error(`uploadPartCopy: Expected success, got error: ${err}`))); + }, + next => { + const completeMpuParams = { + Bucket: bucket, + Key: this.test.mpuKeyNameGcp, + MultipartUpload: { + Parts: [ + { + ETag: `"${fiveMbMD5}"`, + PartNumber: 1, + }, + { + ETag: `"${fiveMbMD5}"`, + PartNumber: 2, + }, + ], + }, + UploadId: this.test.uploadId, + }; + s3.send(new CompleteMultipartUploadCommand(completeMpuParams)) + .then(res => { + assert.strictEqual(res.Bucket, bucket); + assert.strictEqual(res.Key, this.test.mpuKeyNameGcp); + next(); + }) + .catch(err => + next(new Error(`completeMultipartUpload: Expected success, got error: ${err}`)), + ); + }, + ], + done, + ); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/objectTagging/objectTagging.js b/tests/functional/aws-node-sdk/test/multipleBackend/objectTagging/objectTagging.js index 976324d4a6..e36f164413 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/objectTagging/objectTagging.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/objectTagging/objectTagging.js @@ -13,9 +13,19 @@ const { } = require('@aws-sdk/client-s3'); const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); -const { describeSkipIfNotMultiple, awsS3, awsBucket, getAwsRetry, - getAzureClient, getAzureContainerName, convertMD5, memLocation, - fileLocation, awsLocation, azureLocation, genUniqID, +const { + describeSkipIfNotMultiple, + awsS3, + awsBucket, + getAwsRetry, + getAzureClient, + getAzureContainerName, + convertMD5, + memLocation, + fileLocation, + awsLocation, + azureLocation, + genUniqID, } = require('../utils'); const azureClient = getAzureClient(); @@ -52,23 +62,19 @@ const putTags = { const tagObj = { key1: 'value1', key2: 'value2' }; function getAndAssertObjectTags(tagParams, callback) { - return s3.send(new GetObjectTaggingCommand(tagParams)) + return s3 + .send(new GetObjectTaggingCommand(tagParams)) .then(res => { assert.strictEqual(res.TagSet.length, 2); - assert.strictEqual(res.TagSet[0].Key, - putTags.TagSet[0].Key); - assert.strictEqual(res.TagSet[0].Value, - putTags.TagSet[0].Value); - assert.strictEqual(res.TagSet[1].Key, - putTags.TagSet[1].Key); - assert.strictEqual(res.TagSet[1].Value, - putTags.TagSet[1].Value); + assert.strictEqual(res.TagSet[0].Key, putTags.TagSet[0].Key); + assert.strictEqual(res.TagSet[0].Value, putTags.TagSet[0].Value); + assert.strictEqual(res.TagSet[1].Key, putTags.TagSet[1].Key); + assert.strictEqual(res.TagSet[1].Value, putTags.TagSet[1].Value); callback(); }) .catch(callback); } - function awsGet(key, tagCheck, isEmpty, isMpu, callback) { process.stdout.write('Getting object from AWS\n'); getAwsRetry({ key }, 0, (err, res) => { @@ -91,24 +97,29 @@ function awsGet(key, tagCheck, isEmpty, isMpu, callback) { function azureGet(key, tagCheck, isEmpty, callback) { process.stdout.write('Getting object from Azure\n'); - azureClient.getContainerClient(azureContainerName).getProperties(key).then(res => { - const resMD5 = convertMD5(res.contentSettings.contentMD5); - if (isEmpty) { - assert.strictEqual(resMD5, `${emptyMD5}`); - } else { - assert.strictEqual(resMD5, `${correctMD5}`); - } - if (tagCheck) { - assert.strictEqual(res.metadata.tags, - JSON.stringify(tagObj)); - } else { - assert.strictEqual(res.metadata.tags, undefined); - } - return callback(); - }, err => { - assert.equal(err, null); - return callback(); - }); + azureClient + .getContainerClient(azureContainerName) + .getProperties(key) + .then( + res => { + const resMD5 = convertMD5(res.contentSettings.contentMD5); + if (isEmpty) { + assert.strictEqual(resMD5, `${emptyMD5}`); + } else { + assert.strictEqual(resMD5, `${correctMD5}`); + } + if (tagCheck) { + assert.strictEqual(res.metadata.tags, JSON.stringify(tagObj)); + } else { + assert.strictEqual(res.metadata.tags, undefined); + } + return callback(); + }, + err => { + assert.equal(err, null); + return callback(); + }, + ); } function getObject(key, backend, tagCheck, isEmpty, isMpu, callback) { @@ -123,8 +134,7 @@ function getObject(key, backend, tagCheck, isEmpty, isMpu, callback) { } else { assert.strictEqual(res.ETag, `"${correctMD5}"`); } - assert.strictEqual(res.Metadata['scal-location-constraint'], - backend); + assert.strictEqual(res.Metadata['scal-location-constraint'], backend); if (tagCheck) { assert.strictEqual(res.TagCount, 2); } else { @@ -158,37 +168,41 @@ function getObject(key, backend, tagCheck, isEmpty, isMpu, callback) { } function mpuWaterfall(params, cb) { - async.waterfall([ - next => { - s3.send(new CreateMultipartUploadCommand(params)) - .then(data => next(null, data.UploadId)) - .catch(next); - }, - (uploadId, next) => { - const partParams = { Bucket: bucket, Key: params.Key, PartNumber: 1, - UploadId: uploadId, Body: body }; - s3.send(new UploadPartCommand(partParams)) - .then(result => next(null, uploadId, result.ETag)) - .catch(next); - }, - (uploadId, eTag, next) => { - const compParams = { Bucket: bucket, Key: params.Key, - MultipartUpload: { - Parts: [{ ETag: eTag, PartNumber: 1 }], - }, - UploadId: uploadId }; - s3.send(new CompleteMultipartUploadCommand(compParams)) - .then(() => next()) - .catch(next); + async.waterfall( + [ + next => { + s3.send(new CreateMultipartUploadCommand(params)) + .then(data => next(null, data.UploadId)) + .catch(next); + }, + (uploadId, next) => { + const partParams = { Bucket: bucket, Key: params.Key, PartNumber: 1, UploadId: uploadId, Body: body }; + s3.send(new UploadPartCommand(partParams)) + .then(result => next(null, uploadId, result.ETag)) + .catch(next); + }, + (uploadId, eTag, next) => { + const compParams = { + Bucket: bucket, + Key: params.Key, + MultipartUpload: { + Parts: [{ ETag: eTag, PartNumber: 1 }], + }, + UploadId: uploadId, + }; + s3.send(new CompleteMultipartUploadCommand(compParams)) + .then(() => next()) + .catch(next); + }, + ], + err => { + assert.equal(err, null); + cb(err); }, - ], err => { - assert.equal(err, null); - cb(err); - }); + ); } -describeSkipIfNotMultiple('Object tagging with multiple backends', -function testSuite() { +describeSkipIfNotMultiple('Object tagging with multiple backends', function testSuite() { if (!process.env.S3_END_TO_END) { this.retries(2); } @@ -197,8 +211,7 @@ function testSuite() { beforeEach(() => { bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; - return s3.send(new CreateBucketCommand({ Bucket: bucket })) - .catch(err => { + return s3.send(new CreateBucketCommand({ Bucket: bucket })).catch(err => { process.stdout.write(`Error creating bucket: ${err}\n`); throw err; }); @@ -206,26 +219,27 @@ function testSuite() { afterEach(() => { process.stdout.write('Emptying bucket\n'); - return bucketUtil.empty(bucket) - .then(() => { - process.stdout.write('Deleting bucket\n'); - return bucketUtil.deleteOne(bucket); - }) - .catch(err => { - process.stdout.write(`Error in afterEach: ${err}\n`); - throw err; - }); + return bucketUtil + .empty(bucket) + .then(() => { + process.stdout.write('Deleting bucket\n'); + return bucketUtil.deleteOne(bucket); + }) + .catch(err => { + process.stdout.write(`Error in afterEach: ${err}\n`); + throw err; + }); }); describe('putObject with tags and putObjectTagging', () => { testBackends.forEach(backend => { const itSkipIfAzure = backend === 'azurebackend' ? it.skip : it; - it(`should put an object with tags to ${backend} backend`, - done => { + it(`should put an object with tags to ${backend} backend`, done => { const key = `somekey-${genUniqID()}`; - const params = Object.assign({ Key: key, Tagging: tagString, - Metadata: { 'scal-location-constraint': backend } }, - putParams); + const params = Object.assign( + { Key: key, Tagging: tagString, Metadata: { 'scal-location-constraint': backend } }, + putParams, + ); process.stdout.write('Putting object\n'); s3.send(new PutObjectCommand(params)) .then(() => { @@ -234,8 +248,7 @@ function testSuite() { .catch(done); }); - it(`should put a 0 byte object with tags to ${backend} backend`, - done => { + it(`should put a 0 byte object with tags to ${backend} backend`, done => { const key = `somekey-${genUniqID()}`; const params = { Bucket: bucket, @@ -251,16 +264,16 @@ function testSuite() { .catch(done); }); - it(`should put tags to preexisting object in ${backend} ` + - 'backend', done => { + it(`should put tags to preexisting object in ${backend} ` + 'backend', done => { const key = `somekey-${genUniqID()}`; - const params = Object.assign({ Key: key, Metadata: - { 'scal-location-constraint': backend } }, putParams); + const params = Object.assign( + { Key: key, Metadata: { 'scal-location-constraint': backend } }, + putParams, + ); process.stdout.write('Putting object\n'); s3.send(new PutObjectCommand(params)) .then(() => { - const putTagParams = { Bucket: bucket, Key: key, - Tagging: putTags }; + const putTagParams = { Bucket: bucket, Key: key, Tagging: putTags }; process.stdout.write('Putting object tags\n'); return s3.send(new PutObjectTaggingCommand(putTagParams)); }) @@ -270,8 +283,7 @@ function testSuite() { .catch(done); }); - it('should put tags to preexisting 0 byte object in ' + - `${backend} backend`, done => { + it('should put tags to preexisting 0 byte object in ' + `${backend} backend`, done => { const key = `somekey-${genUniqID()}`; const params = { Bucket: bucket, @@ -281,8 +293,7 @@ function testSuite() { process.stdout.write('Putting object\n'); s3.send(new PutObjectCommand(params)) .then(() => { - const putTagParams = { Bucket: bucket, Key: key, - Tagging: putTags }; + const putTagParams = { Bucket: bucket, Key: key, Tagging: putTags }; process.stdout.write('Putting object tags\n'); return s3.send(new PutObjectTaggingCommand(putTagParams)); }) @@ -292,8 +303,7 @@ function testSuite() { .catch(done); }); - itSkipIfAzure('should put tags to completed MPU ' + - `object in ${backend}`, done => { + itSkipIfAzure('should put tags to completed MPU ' + `object in ${backend}`, done => { const key = `somekey-${genUniqID()}`; const params = { Bucket: bucket, @@ -305,8 +315,7 @@ function testSuite() { done(err); return; } - const putTagParams = { Bucket: bucket, Key: key, - Tagging: putTags }; + const putTagParams = { Bucket: bucket, Key: key, Tagging: putTags }; process.stdout.write('Putting object\n'); s3.send(new PutObjectTaggingCommand(putTagParams)) .then(() => { @@ -317,44 +326,50 @@ function testSuite() { }); }); - it('should not return error putting tags to correct object ' + - 'version in AWS, even if a delete marker was created directly ' + - 'on AWS before tags are put', - done => { - const key = `somekey-${genUniqID()}`; - const params = Object.assign({ Key: key, Metadata: - { 'scal-location-constraint': awsLocation } }, putParams); - process.stdout.write('Putting object\n'); - s3.send(new PutObjectCommand(params)) - .then(() => new Promise((resolve, reject) => { - process.stdout.write('Deleting object from AWS\n'); - awsS3.deleteObject({ Bucket: awsBucket, Key: key }, err => { - if (err) { - reject(err); - return; - } - resolve(); - }); - })) - .then(() => { - const putTagParams = { Bucket: bucket, Key: key, - Tagging: putTags }; - process.stdout.write('Putting object tags\n'); - return s3.send(new PutObjectTaggingCommand(putTagParams)); - }) - .then(() => done()) - .catch(done); - }); + it( + 'should not return error putting tags to correct object ' + + 'version in AWS, even if a delete marker was created directly ' + + 'on AWS before tags are put', + done => { + const key = `somekey-${genUniqID()}`; + const params = Object.assign( + { Key: key, Metadata: { 'scal-location-constraint': awsLocation } }, + putParams, + ); + process.stdout.write('Putting object\n'); + s3.send(new PutObjectCommand(params)) + .then( + () => + new Promise((resolve, reject) => { + process.stdout.write('Deleting object from AWS\n'); + awsS3.deleteObject({ Bucket: awsBucket, Key: key }, err => { + if (err) { + reject(err); + return; + } + resolve(); + }); + }), + ) + .then(() => { + const putTagParams = { Bucket: bucket, Key: key, Tagging: putTags }; + process.stdout.write('Putting object tags\n'); + return s3.send(new PutObjectTaggingCommand(putTagParams)); + }) + .then(() => done()) + .catch(done); + }, + ); }); describe('getObjectTagging', () => { testBackends.forEach(backend => { - it(`should get tags from object on ${backend} backend`, - done => { + it(`should get tags from object on ${backend} backend`, done => { const key = `somekey-${genUniqID()}`; - const params = Object.assign({ Key: key, Tagging: tagString, - Metadata: { 'scal-location-constraint': backend } }, - putParams); + const params = Object.assign( + { Key: key, Tagging: tagString, Metadata: { 'scal-location-constraint': backend } }, + putParams, + ); process.stdout.write('Putting object\n'); s3.send(new PutObjectCommand(params)) .then(() => { @@ -365,40 +380,47 @@ function testSuite() { }); }); - it('should not return error on getting tags from object that has ' + - 'had a delete marker put directly on AWS', done => { - const key = `somekey-${genUniqID()}`; - const params = Object.assign({ Key: key, Tagging: tagString, - Metadata: { 'scal-location-constraint': awsLocation } }, - putParams); - process.stdout.write('Putting object\n'); - s3.send(new PutObjectCommand(params)) - .then(() => new Promise((resolve, reject) => { - process.stdout.write('Deleting object from AWS\n'); - awsS3.deleteObject({ Bucket: awsBucket, Key: key }, err => { - if (err) { - reject(err); - return; - } - resolve(); - }); - })) - .then(() => { - const tagParams = { Bucket: bucket, Key: key }; - getAndAssertObjectTags(tagParams, done); - }) - .catch(done); - }); + it( + 'should not return error on getting tags from object that has ' + + 'had a delete marker put directly on AWS', + done => { + const key = `somekey-${genUniqID()}`; + const params = Object.assign( + { Key: key, Tagging: tagString, Metadata: { 'scal-location-constraint': awsLocation } }, + putParams, + ); + process.stdout.write('Putting object\n'); + s3.send(new PutObjectCommand(params)) + .then( + () => + new Promise((resolve, reject) => { + process.stdout.write('Deleting object from AWS\n'); + awsS3.deleteObject({ Bucket: awsBucket, Key: key }, err => { + if (err) { + reject(err); + return; + } + resolve(); + }); + }), + ) + .then(() => { + const tagParams = { Bucket: bucket, Key: key }; + getAndAssertObjectTags(tagParams, done); + }) + .catch(done); + }, + ); }); describe('deleteObjectTagging', () => { testBackends.forEach(backend => { - it(`should delete tags from object on ${backend} backend`, - done => { + it(`should delete tags from object on ${backend} backend`, done => { const key = `somekey-${genUniqID()}`; - const params = Object.assign({ Key: key, Tagging: tagString, - Metadata: { 'scal-location-constraint': backend } }, - putParams); + const params = Object.assign( + { Key: key, Tagging: tagString, Metadata: { 'scal-location-constraint': backend } }, + putParams, + ); process.stdout.write('Putting object\n'); s3.send(new PutObjectCommand(params)) .then(() => { @@ -412,31 +434,38 @@ function testSuite() { }); }); - it('should not return error on deleting tags from object that ' + - 'has had delete markers put directly on AWS', done => { - const key = `somekey-${genUniqID()}`; - const params = Object.assign({ Key: key, Tagging: tagString, - Metadata: { 'scal-location-constraint': awsLocation } }, - putParams); - process.stdout.write('Putting object\n'); - s3.send(new PutObjectCommand(params)) - .then(() => new Promise((resolve, reject) => { - process.stdout.write('Deleting object from AWS\n'); - awsS3.deleteObject({ Bucket: awsBucket, Key: key }, err => { - if (err) { - reject(err); - return; - } - resolve(); - }); - })) - .then(() => { - const tagParams = { Bucket: bucket, Key: key }; - return s3.send(new DeleteObjectTaggingCommand(tagParams)); - }) - .then(() => done()) - .catch(done); - }); + it( + 'should not return error on deleting tags from object that ' + + 'has had delete markers put directly on AWS', + done => { + const key = `somekey-${genUniqID()}`; + const params = Object.assign( + { Key: key, Tagging: tagString, Metadata: { 'scal-location-constraint': awsLocation } }, + putParams, + ); + process.stdout.write('Putting object\n'); + s3.send(new PutObjectCommand(params)) + .then( + () => + new Promise((resolve, reject) => { + process.stdout.write('Deleting object from AWS\n'); + awsS3.deleteObject({ Bucket: awsBucket, Key: key }, err => { + if (err) { + reject(err); + return; + } + resolve(); + }); + }), + ) + .then(() => { + const tagParams = { Bucket: bucket, Key: key }; + return s3.send(new DeleteObjectTaggingCommand(tagParams)); + }) + .then(() => done()) + .catch(done); + }, + ); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/objectTagging/taggingAwsVersioning-delete.js b/tests/functional/aws-node-sdk/test/multipleBackend/objectTagging/taggingAwsVersioning-delete.js index 52a2a91297..812316765d 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/objectTagging/taggingAwsVersioning-delete.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/objectTagging/taggingAwsVersioning-delete.js @@ -15,7 +15,8 @@ const { tagging, genUniqID, } = require('../utils'); -const { PutObjectCommand, +const { + PutObjectCommand, DeleteObjectCommand, CreateBucketCommand, DeleteBucketCommand, @@ -25,213 +26,317 @@ const { putTaggingAndAssert, delTaggingAndAssert, awsGetAssertTags } = tagging; const bucket = `awsversioningtagdel${genUniqID()}`; const someBody = 'teststring'; -describeSkipIfNotMultiple('AWS backend object delete tagging with versioning ', -function testSuite() { +describeSkipIfNotMultiple('AWS backend object delete tagging with versioning ', function testSuite() { this.timeout(120000); const tags = { key1: 'value1', key2: 'value2' }; withV4(sigCfg => { const bucketUtil = new BucketUtility('default', sigCfg); const s3 = bucketUtil.s3; - beforeEach(done => s3.send(new CreateBucketCommand({ - Bucket: bucket, - CreateBucketConfiguration: { - LocationConstraint: awsLocation, - }, - })).then(() => done()).catch(err => done(err))); + beforeEach(done => + s3 + .send( + new CreateBucketCommand({ + Bucket: bucket, + CreateBucketConfiguration: { + LocationConstraint: awsLocation, + }, + }), + ) + .then(() => done()) + .catch(err => done(err)), + ); afterEach(done => { removeAllVersions({ Bucket: bucket }, err => { if (err) { return done(err); } - return s3.send(new DeleteBucketCommand({ Bucket: bucket })) - .then(() => done()).catch(done); + return s3 + .send(new DeleteBucketCommand({ Bucket: bucket })) + .then(() => done()) + .catch(done); }); }); - it('versioning not configured: should delete a tag set on the ' + - 'latest version if no version is specified', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => s3.send(new PutObjectCommand({ Bucket: bucket, Key: key })).then(data => - next(null, data)).catch(err => next(err)), - (putData, next) => putTaggingAndAssert(s3, { bucket, key, tags, - expectedVersionId: false }, next), - (versionId, next) => delTaggingAndAssert(s3, { bucket, key, - expectedVersionId: false }, next), - next => awsGetAssertTags({ key, expectedTags: {} }, next), - ], done); - }); - - it('versioning not configured: should delete a tag set on the ' + - 'version if specified (null)', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => s3.send(new PutObjectCommand({ Bucket: bucket, Key: key })).then(data => - next(null, data)).catch(err => next(err)), - (putData, next) => putTaggingAndAssert(s3, { bucket, key, tags, - versionId: 'null', expectedVersionId: false }, next), - (versionId, next) => delTaggingAndAssert(s3, { bucket, key, - versionId: 'null', expectedVersionId: false }, next), - next => awsGetAssertTags({ key, expectedTags: {} }, next), - ], done); - }); + it( + 'versioning not configured: should delete a tag set on the ' + 'latest version if no version is specified', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => + s3 + .send(new PutObjectCommand({ Bucket: bucket, Key: key })) + .then(data => next(null, data)) + .catch(err => next(err)), + (putData, next) => + putTaggingAndAssert(s3, { bucket, key, tags, expectedVersionId: false }, next), + (versionId, next) => delTaggingAndAssert(s3, { bucket, key, expectedVersionId: false }, next), + next => awsGetAssertTags({ key, expectedTags: {} }, next), + ], + done, + ); + }, + ); - it('versioning suspended: should delete a tag set on the latest ' + - 'version if no version is specified', done => { - const data = [undefined, 'test1', 'test2']; + it('versioning not configured: should delete a tag set on the ' + 'version if specified (null)', done => { const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => putNullVersionsToAws(s3, bucket, key, data, next), - (versionIds, next) => putTaggingAndAssert(s3, { bucket, key, - tags, expectedVersionId: 'null' }, next), - (versionId, next) => delTaggingAndAssert(s3, { bucket, key, - expectedVersionId: 'null' }, next), - next => awsGetAssertTags({ key, expectedTags: {} }, next), - ], done); + async.waterfall( + [ + next => + s3 + .send(new PutObjectCommand({ Bucket: bucket, Key: key })) + .then(data => next(null, data)) + .catch(err => next(err)), + (putData, next) => + putTaggingAndAssert( + s3, + { bucket, key, tags, versionId: 'null', expectedVersionId: false }, + next, + ), + (versionId, next) => + delTaggingAndAssert(s3, { bucket, key, versionId: 'null', expectedVersionId: false }, next), + next => awsGetAssertTags({ key, expectedTags: {} }, next), + ], + done, + ); }); - it('versioning suspended: should delete a tag set on a specific ' + - 'version (null)', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => putNullVersionsToAws(s3, bucket, key, [undefined], - next), - (versionIds, next) => putTaggingAndAssert(s3, { bucket, key, - tags, versionId: 'null', expectedVersionId: 'null' }, next), - (versionId, next) => delTaggingAndAssert(s3, { bucket, key, - versionId: 'null', expectedTags: tags, - expectedVersionId: 'null' }, next), - next => awsGetAssertTags({ key, expectedTags: {} }, next), - ], done); - }); + it( + 'versioning suspended: should delete a tag set on the latest ' + 'version if no version is specified', + done => { + const data = [undefined, 'test1', 'test2']; + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => putNullVersionsToAws(s3, bucket, key, data, next), + (versionIds, next) => + putTaggingAndAssert(s3, { bucket, key, tags, expectedVersionId: 'null' }, next), + (versionId, next) => delTaggingAndAssert(s3, { bucket, key, expectedVersionId: 'null' }, next), + next => awsGetAssertTags({ key, expectedTags: {} }, next), + ], + done, + ); + }, + ); - it('versioning enabled then suspended: should delete a tag set on ' + - 'a specific (non-null) version if specified', done => { + it('versioning suspended: should delete a tag set on a specific ' + 'version (null)', done => { const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => enableVersioning(s3, bucket, next), - next => s3.send(new PutObjectCommand({ Bucket: bucket, Key: key })).then(data => - next(null, data)).catch(err => next(err)), - (putData, next) => awsGetLatestVerId(key, '', - (err, awsVid) => next(err, putData.VersionId, awsVid)), - (s3Vid, awsVid, next) => putNullVersionsToAws(s3, bucket, key, - [someBody], () => next(null, s3Vid, awsVid)), - (s3Vid, awsVid, next) => putTaggingAndAssert(s3, { bucket, key, - tags, versionId: s3Vid, expectedVersionId: s3Vid }, () => - next(null, s3Vid, awsVid)), - (s3Vid, awsVid, next) => delTaggingAndAssert(s3, { bucket, key, - versionId: s3Vid, expectedVersionId: s3Vid }, - () => next(null, awsVid)), - (awsVid, next) => awsGetAssertTags({ key, versionId: awsVid, - expectedTags: {} }, next), - ], done); + async.waterfall( + [ + next => putNullVersionsToAws(s3, bucket, key, [undefined], next), + (versionIds, next) => + putTaggingAndAssert( + s3, + { bucket, key, tags, versionId: 'null', expectedVersionId: 'null' }, + next, + ), + (versionId, next) => + delTaggingAndAssert( + s3, + { bucket, key, versionId: 'null', expectedTags: tags, expectedVersionId: 'null' }, + next, + ), + next => awsGetAssertTags({ key, expectedTags: {} }, next), + ], + done, + ); }); - it('versioning enabled: should delete a tag set on the latest ' + - 'version if no version is specified', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => enableVersioning(s3, bucket, next), - next => s3.send(new PutObjectCommand({ Bucket: bucket, Key: key })).then(putData => - next(null, putData)).catch(err => next(err)), - (putData, next) => putTaggingAndAssert(s3, { bucket, key, tags, - expectedVersionId: putData.VersionId }, next), - (versionId, next) => delTaggingAndAssert(s3, { bucket, key, - expectedVersionId: versionId }, next), - next => awsGetAssertTags({ key, expectedTags: {} }, next), - ], done); - }); + it( + 'versioning enabled then suspended: should delete a tag set on ' + + 'a specific (non-null) version if specified', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => enableVersioning(s3, bucket, next), + next => + s3 + .send(new PutObjectCommand({ Bucket: bucket, Key: key })) + .then(data => next(null, data)) + .catch(err => next(err)), + (putData, next) => + awsGetLatestVerId(key, '', (err, awsVid) => next(err, putData.VersionId, awsVid)), + (s3Vid, awsVid, next) => + putNullVersionsToAws(s3, bucket, key, [someBody], () => next(null, s3Vid, awsVid)), + (s3Vid, awsVid, next) => + putTaggingAndAssert( + s3, + { bucket, key, tags, versionId: s3Vid, expectedVersionId: s3Vid }, + () => next(null, s3Vid, awsVid), + ), + (s3Vid, awsVid, next) => + delTaggingAndAssert(s3, { bucket, key, versionId: s3Vid, expectedVersionId: s3Vid }, () => + next(null, awsVid), + ), + (awsVid, next) => awsGetAssertTags({ key, versionId: awsVid, expectedTags: {} }, next), + ], + done, + ); + }, + ); - it('versioning enabled: should delete a tag set on a specific version', - done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => enableVersioning(s3, bucket, next), - next => s3.send(new PutObjectCommand({ Bucket: bucket, Key: key })).then(putData => - next(null, putData)).catch(err => next(err)), - (putData, next) => putTaggingAndAssert(s3, { bucket, key, tags, - versionId: putData.VersionId, - expectedVersionId: putData.VersionId }, next), - (versionId, next) => delTaggingAndAssert(s3, { bucket, key, - versionId, expectedVersionId: versionId }, next), - next => awsGetAssertTags({ key, expectedTags: {} }, next), - ], done); - }); + it( + 'versioning enabled: should delete a tag set on the latest ' + 'version if no version is specified', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => enableVersioning(s3, bucket, next), + next => + s3 + .send(new PutObjectCommand({ Bucket: bucket, Key: key })) + .then(putData => next(null, putData)) + .catch(err => next(err)), + (putData, next) => + putTaggingAndAssert(s3, { bucket, key, tags, expectedVersionId: putData.VersionId }, next), + (versionId, next) => + delTaggingAndAssert(s3, { bucket, key, expectedVersionId: versionId }, next), + next => awsGetAssertTags({ key, expectedTags: {} }, next), + ], + done, + ); + }, + ); - it('versioning enabled: should delete a tag set on a specific ' + - 'version that is not the latest version', done => { + it('versioning enabled: should delete a tag set on a specific version', done => { const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => enableVersioning(s3, bucket, next), - next => s3.send(new PutObjectCommand({ Bucket: bucket, Key: key })).then(putData => - next(null, putData)).catch(err => next(err)), - (putData, next) => awsGetLatestVerId(key, '', - (err, awsVid) => next(err, putData.VersionId, awsVid)), - // put another version - (s3Vid, awsVid, next) => s3.send(new PutObjectCommand({ Bucket: bucket, - Key: key, Body: someBody })).then(() => - next(null, s3Vid, awsVid)).catch(err => next(err, s3Vid, awsVid)), - (s3Vid, awsVid, next) => putTaggingAndAssert(s3, { bucket, key, - tags, versionId: s3Vid, expectedVersionId: s3Vid }, err => - next(err, s3Vid, awsVid)), - (s3Vid, awsVid, next) => delTaggingAndAssert(s3, { bucket, key, - versionId: s3Vid, expectedVersionId: s3Vid }, - () => next(null, awsVid)), - (awsVid, next) => awsGetAssertTags({ key, versionId: awsVid, - expectedTags: {} }, next), - ], done); + async.waterfall( + [ + next => enableVersioning(s3, bucket, next), + next => + s3 + .send(new PutObjectCommand({ Bucket: bucket, Key: key })) + .then(putData => next(null, putData)) + .catch(err => next(err)), + (putData, next) => + putTaggingAndAssert( + s3, + { bucket, key, tags, versionId: putData.VersionId, expectedVersionId: putData.VersionId }, + next, + ), + (versionId, next) => + delTaggingAndAssert(s3, { bucket, key, versionId, expectedVersionId: versionId }, next), + next => awsGetAssertTags({ key, expectedTags: {} }, next), + ], + done, + ); }); - it('versioning suspended then enabled: should delete a tag set on ' + - 'a specific version (null) if specified', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => putNullVersionsToAws(s3, bucket, key, [undefined], - () => next()), - next => awsGetLatestVerId(key, '', next), - (awsVid, next) => putVersionsToAws(s3, bucket, key, [someBody], - () => next(null, awsVid)), - (awsVid, next) => putTaggingAndAssert(s3, { bucket, key, tags, - versionId: 'null', expectedVersionId: 'null' }, - () => next(null, awsVid)), - (awsVid, next) => delTaggingAndAssert(s3, { bucket, key, - versionId: 'null', expectedVersionId: 'null' }, - () => next(null, awsVid)), - (awsVid, next) => awsGetAssertTags({ key, versionId: awsVid, - expectedTags: {} }, next), - ], done); - }); + it( + 'versioning enabled: should delete a tag set on a specific ' + 'version that is not the latest version', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => enableVersioning(s3, bucket, next), + next => + s3 + .send(new PutObjectCommand({ Bucket: bucket, Key: key })) + .then(putData => next(null, putData)) + .catch(err => next(err)), + (putData, next) => + awsGetLatestVerId(key, '', (err, awsVid) => next(err, putData.VersionId, awsVid)), + // put another version + (s3Vid, awsVid, next) => + s3 + .send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: someBody })) + .then(() => next(null, s3Vid, awsVid)) + .catch(err => next(err, s3Vid, awsVid)), + (s3Vid, awsVid, next) => + putTaggingAndAssert( + s3, + { bucket, key, tags, versionId: s3Vid, expectedVersionId: s3Vid }, + err => next(err, s3Vid, awsVid), + ), + (s3Vid, awsVid, next) => + delTaggingAndAssert(s3, { bucket, key, versionId: s3Vid, expectedVersionId: s3Vid }, () => + next(null, awsVid), + ), + (awsVid, next) => awsGetAssertTags({ key, versionId: awsVid, expectedTags: {} }, next), + ], + done, + ); + }, + ); - it('should return an ServiceUnavailable if trying to delete ' + - 'tags from object that was deleted from AWS directly', - done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => s3.send(new PutObjectCommand({ Bucket: bucket, Key: key })).then(putData => - next(null, putData)).catch(err => next(err)), - (putData, next) => awsGetLatestVerId(key, '', next), - (awsVid, next) => awsS3.send(new DeleteObjectCommand({ Bucket: awsBucket, - Key: key, VersionId: awsVid })).then(delData => next(null, delData)).catch(err => next(err)), - (delData, next) => delTaggingAndAssert(s3, { bucket, key, - expectedError: 'ServiceUnavailable' }, next), - ], done); - }); + it( + 'versioning suspended then enabled: should delete a tag set on ' + 'a specific version (null) if specified', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => putNullVersionsToAws(s3, bucket, key, [undefined], () => next()), + next => awsGetLatestVerId(key, '', next), + (awsVid, next) => putVersionsToAws(s3, bucket, key, [someBody], () => next(null, awsVid)), + (awsVid, next) => + putTaggingAndAssert( + s3, + { bucket, key, tags, versionId: 'null', expectedVersionId: 'null' }, + () => next(null, awsVid), + ), + (awsVid, next) => + delTaggingAndAssert(s3, { bucket, key, versionId: 'null', expectedVersionId: 'null' }, () => + next(null, awsVid), + ), + (awsVid, next) => awsGetAssertTags({ key, versionId: awsVid, expectedTags: {} }, next), + ], + done, + ); + }, + ); - it('should return an ServiceUnavailable if trying to delete ' + - 'tags from object that was deleted from AWS directly', - done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => s3.send(new PutObjectCommand({ Bucket: bucket, Key: key })).then(putData => - next(null, putData)).catch(err => next(err)), - (putData, next) => awsGetLatestVerId(key, '', next), - (awsVid, next) => awsS3.send(new DeleteObjectCommand({ Bucket: awsBucket, - Key: key, VersionId: awsVid })).then(delData => next(null, delData)).catch(err => next(err)), - (delData, next) => delTaggingAndAssert(s3, { bucket, key, - expectedError: 'ServiceUnavailable' }, next), - ], done); - }); + it( + 'should return an ServiceUnavailable if trying to delete ' + + 'tags from object that was deleted from AWS directly', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => + s3 + .send(new PutObjectCommand({ Bucket: bucket, Key: key })) + .then(putData => next(null, putData)) + .catch(err => next(err)), + (putData, next) => awsGetLatestVerId(key, '', next), + (awsVid, next) => + awsS3 + .send(new DeleteObjectCommand({ Bucket: awsBucket, Key: key, VersionId: awsVid })) + .then(delData => next(null, delData)) + .catch(err => next(err)), + (delData, next) => + delTaggingAndAssert(s3, { bucket, key, expectedError: 'ServiceUnavailable' }, next), + ], + done, + ); + }, + ); + it( + 'should return an ServiceUnavailable if trying to delete ' + + 'tags from object that was deleted from AWS directly', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => + s3 + .send(new PutObjectCommand({ Bucket: bucket, Key: key })) + .then(putData => next(null, putData)) + .catch(err => next(err)), + (putData, next) => awsGetLatestVerId(key, '', next), + (awsVid, next) => + awsS3 + .send(new DeleteObjectCommand({ Bucket: awsBucket, Key: key, VersionId: awsVid })) + .then(delData => next(null, delData)) + .catch(err => next(err)), + (delData, next) => + delTaggingAndAssert(s3, { bucket, key, expectedError: 'ServiceUnavailable' }, next), + ], + done, + ); + }, + ); }); }); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/objectTagging/taggingAwsVersioning-putget.js b/tests/functional/aws-node-sdk/test/multipleBackend/objectTagging/taggingAwsVersioning-putget.js index 2906707b25..ba74cc41c9 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/objectTagging/taggingAwsVersioning-putget.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/objectTagging/taggingAwsVersioning-putget.js @@ -23,20 +23,18 @@ const { genUniqID, } = require('../utils'); -const { putTaggingAndAssert, getTaggingAndAssert, delTaggingAndAssert, - awsGetAssertTags } = tagging; +const { putTaggingAndAssert, getTaggingAndAssert, delTaggingAndAssert, awsGetAssertTags } = tagging; const bucket = `awsversioningtag${genUniqID()}`; const someBody = 'teststring'; -describeSkipIfNotMultiple('AWS backend object put/get tagging with versioning', -function testSuite() { +describeSkipIfNotMultiple('AWS backend object put/get tagging with versioning', function testSuite() { this.timeout(120000); const tags = { key1: 'value1', key2: 'value2' }; withV4(sigCfg => { const bucketUtil = new BucketUtility('default', sigCfg); const s3 = bucketUtil.s3; - + beforeEach(done => { const command = new CreateBucketCommand({ Bucket: bucket, @@ -54,341 +52,470 @@ function testSuite() { if (err) { return done(err); } - return s3.send(new DeleteBucketCommand({ Bucket: bucket })) - .then(() => done()).catch(done); + return s3 + .send(new DeleteBucketCommand({ Bucket: bucket })) + .then(() => done()) + .catch(done); }); }); - it('versioning not configured: should put/get a tag set on the ' + - 'latest version if no version is specified', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => { - const command = new PutObjectCommand({ Bucket: bucket, Key: key }); - s3.send(command) - .then(data => next(null, data)) - .catch(err => next(err)); - }, - (putData, next) => putTaggingAndAssert(s3, { bucket, key, tags, - expectedVersionId: false }, next), - (versionId, next) => getTaggingAndAssert(s3, { bucket, key, - expectedTags: tags, expectedVersionId: false }, next), - (versionId, next) => awsGetAssertTags({ key, - expectedTags: tags }, next), - ], done); - }); + it( + 'versioning not configured: should put/get a tag set on the ' + 'latest version if no version is specified', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => { + const command = new PutObjectCommand({ Bucket: bucket, Key: key }); + s3.send(command) + .then(data => next(null, data)) + .catch(err => next(err)); + }, + (putData, next) => + putTaggingAndAssert(s3, { bucket, key, tags, expectedVersionId: false }, next), + (versionId, next) => + getTaggingAndAssert( + s3, + { bucket, key, expectedTags: tags, expectedVersionId: false }, + next, + ), + (versionId, next) => awsGetAssertTags({ key, expectedTags: tags }, next), + ], + done, + ); + }, + ); - it('versioning not configured: should put/get a tag set on a ' + - 'specific version if specified (null)', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => { - const command = new PutObjectCommand({ Bucket: bucket, Key: key }); - s3.send(command) - .then(data => next(null, data)) - .catch(err => next(err)); - }, - (putData, next) => putTaggingAndAssert(s3, { bucket, key, tags, - versionId: 'null', expectedVersionId: false }, next), - (versionId, next) => getTaggingAndAssert(s3, { bucket, key, - versionId: 'null', expectedTags: tags, - expectedVersionId: false }, next), - (versionId, next) => awsGetAssertTags({ key, - expectedTags: tags }, next), - ], done); - }); + it( + 'versioning not configured: should put/get a tag set on a ' + 'specific version if specified (null)', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => { + const command = new PutObjectCommand({ Bucket: bucket, Key: key }); + s3.send(command) + .then(data => next(null, data)) + .catch(err => next(err)); + }, + (putData, next) => + putTaggingAndAssert( + s3, + { bucket, key, tags, versionId: 'null', expectedVersionId: false }, + next, + ), + (versionId, next) => + getTaggingAndAssert( + s3, + { bucket, key, versionId: 'null', expectedTags: tags, expectedVersionId: false }, + next, + ), + (versionId, next) => awsGetAssertTags({ key, expectedTags: tags }, next), + ], + done, + ); + }, + ); - it('versioning suspended: should put/get a tag set on the latest ' + - 'version if no version is specified', done => { - const data = [undefined, 'test1', 'test2']; - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => putNullVersionsToAws(s3, bucket, key, data, next), - (versionIds, next) => putTaggingAndAssert(s3, { bucket, key, - tags, expectedVersionId: 'null' }, next), - (versionId, next) => getTaggingAndAssert(s3, { bucket, key, - expectedTags: tags, expectedVersionId: 'null' }, next), - (versionId, next) => awsGetAssertTags({ key, - expectedTags: tags }, next), - ], done); - }); + it( + 'versioning suspended: should put/get a tag set on the latest ' + 'version if no version is specified', + done => { + const data = [undefined, 'test1', 'test2']; + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => putNullVersionsToAws(s3, bucket, key, data, next), + (versionIds, next) => + putTaggingAndAssert(s3, { bucket, key, tags, expectedVersionId: 'null' }, next), + (versionId, next) => + getTaggingAndAssert( + s3, + { bucket, key, expectedTags: tags, expectedVersionId: 'null' }, + next, + ), + (versionId, next) => awsGetAssertTags({ key, expectedTags: tags }, next), + ], + done, + ); + }, + ); - it('versioning suspended: should put/get a tag set on a specific ' + - 'version (null)', done => { + it('versioning suspended: should put/get a tag set on a specific ' + 'version (null)', done => { const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => putNullVersionsToAws(s3, bucket, key, [undefined], - next), - (versionIds, next) => putTaggingAndAssert(s3, { bucket, key, - tags, versionId: 'null', expectedVersionId: 'null' }, next), - (versionId, next) => getTaggingAndAssert(s3, { bucket, key, - versionId: 'null', expectedTags: tags, - expectedVersionId: 'null' }, next), - (versionId, next) => awsGetAssertTags({ key, - expectedTags: tags }, next), - ], done); + async.waterfall( + [ + next => putNullVersionsToAws(s3, bucket, key, [undefined], next), + (versionIds, next) => + putTaggingAndAssert( + s3, + { bucket, key, tags, versionId: 'null', expectedVersionId: 'null' }, + next, + ), + (versionId, next) => + getTaggingAndAssert( + s3, + { bucket, key, versionId: 'null', expectedTags: tags, expectedVersionId: 'null' }, + next, + ), + (versionId, next) => awsGetAssertTags({ key, expectedTags: tags }, next), + ], + done, + ); }); - it('versioning enabled then suspended: should put/get a tag set on ' + - 'a specific (non-null) version if specified', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => enableVersioning(s3, bucket, next), - next => { - const command = new PutObjectCommand({ Bucket: bucket, Key: key }); - s3.send(command) - .then(data => next(null, data)) - .catch(err => next(err)); - }, - (putData, next) => awsGetLatestVerId(key, '', - (err, awsVid) => next(err, putData.VersionId, awsVid)), - (s3Vid, awsVid, next) => putNullVersionsToAws(s3, bucket, key, - [someBody], () => next(null, s3Vid, awsVid)), - (s3Vid, awsVid, next) => putTaggingAndAssert(s3, { bucket, key, - tags, versionId: s3Vid, expectedVersionId: s3Vid }, () => - next(null, s3Vid, awsVid)), - (s3Vid, awsVid, next) => getTaggingAndAssert(s3, { bucket, key, - versionId: s3Vid, expectedTags: tags, - expectedVersionId: s3Vid }, () => next(null, awsVid)), - (awsVid, next) => awsGetAssertTags({ key, versionId: awsVid, - expectedTags: tags }, next), - ], done); - }); + it( + 'versioning enabled then suspended: should put/get a tag set on ' + + 'a specific (non-null) version if specified', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => enableVersioning(s3, bucket, next), + next => { + const command = new PutObjectCommand({ Bucket: bucket, Key: key }); + s3.send(command) + .then(data => next(null, data)) + .catch(err => next(err)); + }, + (putData, next) => + awsGetLatestVerId(key, '', (err, awsVid) => next(err, putData.VersionId, awsVid)), + (s3Vid, awsVid, next) => + putNullVersionsToAws(s3, bucket, key, [someBody], () => next(null, s3Vid, awsVid)), + (s3Vid, awsVid, next) => + putTaggingAndAssert( + s3, + { bucket, key, tags, versionId: s3Vid, expectedVersionId: s3Vid }, + () => next(null, s3Vid, awsVid), + ), + (s3Vid, awsVid, next) => + getTaggingAndAssert( + s3, + { bucket, key, versionId: s3Vid, expectedTags: tags, expectedVersionId: s3Vid }, + () => next(null, awsVid), + ), + (awsVid, next) => awsGetAssertTags({ key, versionId: awsVid, expectedTags: tags }, next), + ], + done, + ); + }, + ); - it('versioning enabled: should put/get a tag set on the latest ' + - 'version if no version is specified', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => enableVersioning(s3, bucket, next), - next => { - const command = new PutObjectCommand({ Bucket: bucket, Key: key }); - s3.send(command) - .then(data => next(null, data)) - .catch(err => next(err)); - }, - (putData, next) => putTaggingAndAssert(s3, { bucket, key, tags, - expectedVersionId: putData.VersionId }, next), - (versionId, next) => getTaggingAndAssert(s3, { bucket, key, - expectedTags: tags, expectedVersionId: versionId }, next), - (versionId, next) => awsGetAssertTags({ key, - expectedTags: tags }, next), - ], done); - }); + it( + 'versioning enabled: should put/get a tag set on the latest ' + 'version if no version is specified', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => enableVersioning(s3, bucket, next), + next => { + const command = new PutObjectCommand({ Bucket: bucket, Key: key }); + s3.send(command) + .then(data => next(null, data)) + .catch(err => next(err)); + }, + (putData, next) => + putTaggingAndAssert(s3, { bucket, key, tags, expectedVersionId: putData.VersionId }, next), + (versionId, next) => + getTaggingAndAssert( + s3, + { bucket, key, expectedTags: tags, expectedVersionId: versionId }, + next, + ), + (versionId, next) => awsGetAssertTags({ key, expectedTags: tags }, next), + ], + done, + ); + }, + ); - it('versioning enabled: should put/get a tag set on a specific version', - done => { + it('versioning enabled: should put/get a tag set on a specific version', done => { const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => enableVersioning(s3, bucket, next), - next => { - const command = new PutObjectCommand({ Bucket: bucket, Key: key }); - s3.send(command) - .then(data => next(null, data)) - .catch(err => next(err)); - }, - (putData, next) => putTaggingAndAssert(s3, { bucket, key, tags, - versionId: putData.VersionId, - expectedVersionId: putData.VersionId }, next), - (versionId, next) => getTaggingAndAssert(s3, { bucket, key, - versionId, expectedTags: tags, - expectedVersionId: versionId }, next), - (versionId, next) => awsGetAssertTags({ key, - expectedTags: tags }, next), - ], done); + async.waterfall( + [ + next => enableVersioning(s3, bucket, next), + next => { + const command = new PutObjectCommand({ Bucket: bucket, Key: key }); + s3.send(command) + .then(data => next(null, data)) + .catch(err => next(err)); + }, + (putData, next) => + putTaggingAndAssert( + s3, + { bucket, key, tags, versionId: putData.VersionId, expectedVersionId: putData.VersionId }, + next, + ), + (versionId, next) => + getTaggingAndAssert( + s3, + { bucket, key, versionId, expectedTags: tags, expectedVersionId: versionId }, + next, + ), + (versionId, next) => awsGetAssertTags({ key, expectedTags: tags }, next), + ], + done, + ); }); - it('versioning enabled: should put/get a tag set on a specific version', - done => { + it('versioning enabled: should put/get a tag set on a specific version', done => { const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => enableVersioning(s3, bucket, next), - next => { - const command = new PutObjectCommand({ Bucket: bucket, Key: key }); - s3.send(command) - .then(data => next(null, data)) - .catch(err => next(err)); - }, - (putData, next) => putTaggingAndAssert(s3, { bucket, key, tags, - versionId: putData.VersionId, - expectedVersionId: putData.VersionId }, next), - (versionId, next) => delTaggingAndAssert(s3, { bucket, key, - versionId, expectedVersionId: versionId }, next), - next => awsGetAssertTags({ key, expectedTags: {} }, next), - ], done); + async.waterfall( + [ + next => enableVersioning(s3, bucket, next), + next => { + const command = new PutObjectCommand({ Bucket: bucket, Key: key }); + s3.send(command) + .then(data => next(null, data)) + .catch(err => next(err)); + }, + (putData, next) => + putTaggingAndAssert( + s3, + { bucket, key, tags, versionId: putData.VersionId, expectedVersionId: putData.VersionId }, + next, + ), + (versionId, next) => + delTaggingAndAssert(s3, { bucket, key, versionId, expectedVersionId: versionId }, next), + next => awsGetAssertTags({ key, expectedTags: {} }, next), + ], + done, + ); }); - it('versioning enabled: should put/get a tag set on a specific ' + - 'version that is not the latest version', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => enableVersioning(s3, bucket, next), - next => { - const command = new PutObjectCommand({ Bucket: bucket, Key: key }); - s3.send(command) - .then(data => next(null, data)) - .catch(err => next(err)); - }, - (putData, next) => awsGetLatestVerId(key, '', - (err, awsVid) => next(err, putData.VersionId, awsVid)), - // put another version - (s3Vid, awsVid, next) => { - const command = new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: someBody - }); - s3.send(command) - .then(() => next(null, s3Vid, awsVid)) - .catch(err => next(err, s3Vid, awsVid)); - }, - (s3Vid, awsVid, next) => putTaggingAndAssert(s3, { bucket, key, - tags, versionId: s3Vid, expectedVersionId: s3Vid }, err => - next(err, s3Vid, awsVid)), - (s3Vid, awsVid, next) => getTaggingAndAssert(s3, { bucket, key, - versionId: s3Vid, expectedTags: tags, - expectedVersionId: s3Vid }, () => next(null, awsVid)), - (awsVid, next) => awsGetAssertTags({ key, versionId: awsVid, - expectedTags: tags }, next), - ], done); - }); + it( + 'versioning enabled: should put/get a tag set on a specific ' + 'version that is not the latest version', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => enableVersioning(s3, bucket, next), + next => { + const command = new PutObjectCommand({ Bucket: bucket, Key: key }); + s3.send(command) + .then(data => next(null, data)) + .catch(err => next(err)); + }, + (putData, next) => + awsGetLatestVerId(key, '', (err, awsVid) => next(err, putData.VersionId, awsVid)), + // put another version + (s3Vid, awsVid, next) => { + const command = new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: someBody, + }); + s3.send(command) + .then(() => next(null, s3Vid, awsVid)) + .catch(err => next(err, s3Vid, awsVid)); + }, + (s3Vid, awsVid, next) => + putTaggingAndAssert( + s3, + { bucket, key, tags, versionId: s3Vid, expectedVersionId: s3Vid }, + err => next(err, s3Vid, awsVid), + ), + (s3Vid, awsVid, next) => + getTaggingAndAssert( + s3, + { bucket, key, versionId: s3Vid, expectedTags: tags, expectedVersionId: s3Vid }, + () => next(null, awsVid), + ), + (awsVid, next) => awsGetAssertTags({ key, versionId: awsVid, expectedTags: tags }, next), + ], + done, + ); + }, + ); - it('versioning suspended then enabled: should put/get a tag set on ' + - 'a specific version (null) if specified', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => putNullVersionsToAws(s3, bucket, key, [undefined], - () => next()), - next => awsGetLatestVerId(key, '', next), - (awsVid, next) => putVersionsToAws(s3, bucket, key, [someBody], - () => next(null, awsVid)), - (awsVid, next) => putTaggingAndAssert(s3, { bucket, key, tags, - versionId: 'null', expectedVersionId: 'null' }, - () => next(null, awsVid)), - (awsVid, next) => getTaggingAndAssert(s3, { bucket, key, - versionId: 'null', expectedTags: tags, - expectedVersionId: 'null' }, () => next(null, awsVid)), - (awsVid, next) => awsGetAssertTags({ key, versionId: awsVid, - expectedTags: tags }, next), - ], done); - }); + it( + 'versioning suspended then enabled: should put/get a tag set on ' + + 'a specific version (null) if specified', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => putNullVersionsToAws(s3, bucket, key, [undefined], () => next()), + next => awsGetLatestVerId(key, '', next), + (awsVid, next) => putVersionsToAws(s3, bucket, key, [someBody], () => next(null, awsVid)), + (awsVid, next) => + putTaggingAndAssert( + s3, + { bucket, key, tags, versionId: 'null', expectedVersionId: 'null' }, + () => next(null, awsVid), + ), + (awsVid, next) => + getTaggingAndAssert( + s3, + { bucket, key, versionId: 'null', expectedTags: tags, expectedVersionId: 'null' }, + () => next(null, awsVid), + ), + (awsVid, next) => awsGetAssertTags({ key, versionId: awsVid, expectedTags: tags }, next), + ], + done, + ); + }, + ); - it('should get tags for an object even if it was deleted from ' + - 'AWS directly (we rely on s3 metadata)', - done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => { - const command = new PutObjectCommand({ Bucket: bucket, Key: key }); - s3.send(command) - .then(data => next(null, data)) - .catch(err => next(err)); - }, - (putData, next) => awsGetLatestVerId(key, '', next), - (awsVid, next) => putTaggingAndAssert(s3, { bucket, key, tags, - expectedVersionId: false }, () => next(null, awsVid)), - (awsVid, next) => { - const command = new DeleteObjectCommand({ - Bucket: awsBucket, - Key: key, - VersionId: awsVid - }); - awsS3.send(command) - .then(data => next(null, data)) - .catch(err => next(err)); - }, - (delData, next) => getTaggingAndAssert(s3, { bucket, key, - expectedTags: tags, expectedVersionId: false, - getObject: false }, next), - ], done); - }); + it( + 'should get tags for an object even if it was deleted from ' + 'AWS directly (we rely on s3 metadata)', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => { + const command = new PutObjectCommand({ Bucket: bucket, Key: key }); + s3.send(command) + .then(data => next(null, data)) + .catch(err => next(err)); + }, + (putData, next) => awsGetLatestVerId(key, '', next), + (awsVid, next) => + putTaggingAndAssert(s3, { bucket, key, tags, expectedVersionId: false }, () => + next(null, awsVid), + ), + (awsVid, next) => { + const command = new DeleteObjectCommand({ + Bucket: awsBucket, + Key: key, + VersionId: awsVid, + }); + awsS3 + .send(command) + .then(data => next(null, data)) + .catch(err => next(err)); + }, + (delData, next) => + getTaggingAndAssert( + s3, + { bucket, key, expectedTags: tags, expectedVersionId: false, getObject: false }, + next, + ), + ], + done, + ); + }, + ); - it('should return an ServiceUnavailable if trying to put ' + - 'tags from object that was deleted from AWS directly', - done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => { - const command = new PutObjectCommand({ Bucket: bucket, Key: key }); - s3.send(command) - .then(data => next(null, data)) - .catch(err => next(err)); - }, - (putData, next) => awsGetLatestVerId(key, '', next), - (awsVid, next) => { - const command = new DeleteObjectCommand({ - Bucket: awsBucket, - Key: key, - VersionId: awsVid - }); - awsS3.send(command) - .then(data => next(null, data)) - .catch(err => next(err)); - }, - (delData, next) => putTaggingAndAssert(s3, { bucket, key, tags, - expectedError: 'ServiceUnavailable' }, next), - ], done); - }); + it( + 'should return an ServiceUnavailable if trying to put ' + + 'tags from object that was deleted from AWS directly', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => { + const command = new PutObjectCommand({ Bucket: bucket, Key: key }); + s3.send(command) + .then(data => next(null, data)) + .catch(err => next(err)); + }, + (putData, next) => awsGetLatestVerId(key, '', next), + (awsVid, next) => { + const command = new DeleteObjectCommand({ + Bucket: awsBucket, + Key: key, + VersionId: awsVid, + }); + awsS3 + .send(command) + .then(data => next(null, data)) + .catch(err => next(err)); + }, + (delData, next) => + putTaggingAndAssert(s3, { bucket, key, tags, expectedError: 'ServiceUnavailable' }, next), + ], + done, + ); + }, + ); - it('should get tags for an version even if it was deleted from ' + - 'AWS directly (we rely on s3 metadata)', - done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => enableVersioning(s3, bucket, next), - next => { - const command = new PutObjectCommand({ Bucket: bucket, Key: key }); - s3.send(command) - .then(data => next(null, data)) - .catch(err => next(err)); - }, - (putData, next) => awsGetLatestVerId(key, '', - (err, awsVid) => next(err, putData.VersionId, awsVid)), - (s3Vid, awsVid, next) => putTaggingAndAssert(s3, { bucket, key, - tags, versionId: s3Vid, expectedVersionId: s3Vid }, - () => next(null, s3Vid, awsVid)), - (s3Vid, awsVid, next) => { - const command = new DeleteObjectCommand({ - Bucket: awsBucket, - Key: key, - VersionId: awsVid - }); - awsS3.send(command) - .then(() => next(null, s3Vid)) - .catch(err => next(err, s3Vid)); - }, - (s3Vid, next) => getTaggingAndAssert(s3, { bucket, key, - versionId: s3Vid, expectedTags: tags, - expectedVersionId: s3Vid, getObject: false }, next), - ], done); - }); + it( + 'should get tags for an version even if it was deleted from ' + 'AWS directly (we rely on s3 metadata)', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => enableVersioning(s3, bucket, next), + next => { + const command = new PutObjectCommand({ Bucket: bucket, Key: key }); + s3.send(command) + .then(data => next(null, data)) + .catch(err => next(err)); + }, + (putData, next) => + awsGetLatestVerId(key, '', (err, awsVid) => next(err, putData.VersionId, awsVid)), + (s3Vid, awsVid, next) => + putTaggingAndAssert( + s3, + { bucket, key, tags, versionId: s3Vid, expectedVersionId: s3Vid }, + () => next(null, s3Vid, awsVid), + ), + (s3Vid, awsVid, next) => { + const command = new DeleteObjectCommand({ + Bucket: awsBucket, + Key: key, + VersionId: awsVid, + }); + awsS3 + .send(command) + .then(() => next(null, s3Vid)) + .catch(err => next(err, s3Vid)); + }, + (s3Vid, next) => + getTaggingAndAssert( + s3, + { + bucket, + key, + versionId: s3Vid, + expectedTags: tags, + expectedVersionId: s3Vid, + getObject: false, + }, + next, + ), + ], + done, + ); + }, + ); - it('should return an ServiceUnavailable if trying to put ' + - 'tags on version that was deleted from AWS directly', - done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => { - const command = new PutObjectCommand({ Bucket: bucket, Key: key }); - s3.send(command) - .then(data => next(null, data)) - .catch(err => next(err)); - }, - (putData, next) => awsGetLatestVerId(key, '', - (err, awsVid) => next(err, putData.VersionId, awsVid)), - (s3Vid, awsVid, next) => { - const command = new DeleteObjectCommand({ - Bucket: awsBucket, - Key: key, - VersionId: awsVid - }); - awsS3.send(command) - .then(() => next(null, s3Vid)) - .catch(err => next(err, s3Vid)); - }, - (s3Vid, next) => putTaggingAndAssert(s3, { bucket, key, tags, - versionId: s3Vid, expectedError: - 'ServiceUnavailable' }, next), - ], done); - }); + it( + 'should return an ServiceUnavailable if trying to put ' + + 'tags on version that was deleted from AWS directly', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => { + const command = new PutObjectCommand({ Bucket: bucket, Key: key }); + s3.send(command) + .then(data => next(null, data)) + .catch(err => next(err)); + }, + (putData, next) => + awsGetLatestVerId(key, '', (err, awsVid) => next(err, putData.VersionId, awsVid)), + (s3Vid, awsVid, next) => { + const command = new DeleteObjectCommand({ + Bucket: awsBucket, + Key: key, + VersionId: awsVid, + }); + awsS3 + .send(command) + .then(() => next(null, s3Vid)) + .catch(err => next(err, s3Vid)); + }, + (s3Vid, next) => + putTaggingAndAssert( + s3, + { bucket, key, tags, versionId: s3Vid, expectedError: 'ServiceUnavailable' }, + next, + ), + ], + done, + ); + }, + ); }); }); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/put/put.js b/tests/functional/aws-node-sdk/test/multipleBackend/put/put.js index 66a5c81f07..0f198381ac 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/put/put.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/put/put.js @@ -11,13 +11,18 @@ const { const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); const { config } = require('../../../../../../lib/Config'); -const { createEncryptedBucketPromise } = - require('../../../lib/utility/createEncryptedBucket'); +const { createEncryptedBucketPromise } = require('../../../lib/utility/createEncryptedBucket'); const { versioningEnabled } = require('../../../lib/utility/versioning-util'); -const { describeSkipIfNotMultiple, getAwsRetry, awsLocation, - awsLocationEncryption, memLocation, fileLocation, genUniqID } - = require('../utils'); +const { + describeSkipIfNotMultiple, + getAwsRetry, + awsLocation, + awsLocationEncryption, + memLocation, + fileLocation, + genUniqID, +} = require('../utils'); const bucket = `putaws${genUniqID()}`; const body = Buffer.from('I am a body', 'utf8'); const bigBody = Buffer.alloc(10485760); @@ -37,7 +42,7 @@ async function getAwsSuccess(key, awsMD5, location) { reject(new Error(`Expected success, got error on direct AWS call: ${err}`)); return; } - + if (location === awsLocationEncryption) { // doesn't check ETag because it's different // with every PUT with encryption @@ -46,8 +51,7 @@ async function getAwsSuccess(key, awsMD5, location) { if (process.env.ENABLE_KMS_ENCRYPTION !== 'true') { assert.strictEqual(res.ETag, `"${awsMD5}"`); } - assert.strictEqual(res.Metadata['scal-location-constraint'], - location); + assert.strictEqual(res.Metadata['scal-location-constraint'], location); resolve(res); }); }); @@ -57,8 +61,7 @@ async function getAwsError(key, expectedError) { return new Promise((resolve, reject) => { getAwsRetry({ key }, 0, err => { try { - assert.notStrictEqual(err, undefined, - 'Expected error but did not find one'); + assert.notStrictEqual(err, undefined, 'Expected error but did not find one'); assert.strictEqual(err.name, expectedError); resolve(); } catch (assertionError) { @@ -71,16 +74,15 @@ async function getAwsError(key, expectedError) { async function awsGetCheck(objectKey, s3MD5, awsMD5, location) { const res = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: objectKey })); assert.strictEqual(res.ETag, `"${s3MD5}"`); - + if (process.env.ENABLE_KMS_ENCRYPTION === 'true') { assert.strictEqual(res.ServerSideEncryption, 'AES256'); } - + process.stdout.write('Getting object from AWS\n'); return await getAwsSuccess(objectKey, awsMD5, location); } - describeSkipIfNotMultiple('MultipleBackend put object', function testSuite() { this.timeout(250000); withV4(sigCfg => { @@ -88,7 +90,7 @@ describeSkipIfNotMultiple('MultipleBackend put object', function testSuite() { bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; process.stdout.write('Creating bucket\n'); - + if (process.env.ENABLE_KMS_ENCRYPTION === 'true') { await createEncryptedBucketPromise({ Bucket: bucket }); } else { @@ -103,301 +105,386 @@ describeSkipIfNotMultiple('MultipleBackend put object', function testSuite() { }); // aws-sdk now (v2.363.0) returns 'UriParameterError' error - it.skip('should return an error to put request without a valid ' + - 'bucket name', - async () => { - const key = `somekey-${genUniqID()}`; - try { - await s3.send(new PutObjectCommand({ Bucket: '', Key: key })); - throw new Error('Expected failure but got success'); - } catch (err) { - assert.strictEqual(err.code, 'MethodNotAllowed'); - } - }); - - describeSkipIfNotMultiple('with set location from "x-amz-meta-scal-' + - 'location-constraint" header', function describe() { - if (!process.env.S3_END_TO_END) { - this.retries(2); + it.skip('should return an error to put request without a valid ' + 'bucket name', async () => { + const key = `somekey-${genUniqID()}`; + try { + await s3.send(new PutObjectCommand({ Bucket: '', Key: key })); + throw new Error('Expected failure but got success'); + } catch (err) { + assert.strictEqual(err.code, 'MethodNotAllowed'); } + }); - it('should return an error to put request without a valid ' + - 'location constraint', async () => { - const key = `somekey-${genUniqID()}`; - const params = { Bucket: bucket, Key: key, - Body: body, - Metadata: { 'scal-location-constraint': 'fail-region' } }; - try { - await s3.send(new PutObjectCommand(params)); - throw new Error('Expected failure but got success'); - } catch (err) { - assert.strictEqual(err.code, 'InvalidArgument'); + describeSkipIfNotMultiple( + 'with set location from "x-amz-meta-scal-' + 'location-constraint" header', + function describe() { + if (!process.env.S3_END_TO_END) { + this.retries(2); } - }); - it('should put an object to mem', async () => { - const key = `somekey-${genUniqID()}`; - const params = { Bucket: bucket, Key: key, - Body: body, - Metadata: { 'scal-location-constraint': memLocation }, - }; - - await s3.send(new PutObjectCommand(params)).then(() => { - process.stdout.write('Putting object succeeded\n'); - }).catch(err => { - throw new Error(`Expected success, got error: ${err}`); - }); - const res = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key })); - assert.strictEqual(res.ETag, `"${correctMD5}"`); - }); - - it('should put a 0-byte object to mem', async () => { - const key = `somekey-${genUniqID()}`; - const params = { Bucket: bucket, Key: key, - Metadata: { 'scal-location-constraint': memLocation }, - }; - - await s3.send(new PutObjectCommand(params)).then(() => { - process.stdout.write('Putting object succeeded\n'); - }).catch(err => { - throw new Error(`Expected success, got error: ${err}`); - }); - const res = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key })); - assert.strictEqual(res.ETag, `"${emptyMD5}"`); - }); - - it('should put only metadata to mem with mdonly header', async () => { - const key = `mdonly-${genUniqID()}`; - const b64 = Buffer.from(correctMD5, 'hex').toString('base64'); - const params = { Bucket: bucket, Key: key, - Metadata: { 'scal-location-constraint': awsLocation, - 'mdonly': 'true', - 'md5chksum': b64, - 'size': body.length.toString(), - } }; - await s3.send(new PutObjectCommand(params)).then(() => { - process.stdout.write('Putting object succeeded\n'); - }).catch(err => { - throw new Error(`Expected success, got error: ${err}`); - }); - const res = await s3.send(new HeadObjectCommand({ Bucket: bucket, Key: key })); - assert.strictEqual(res.ETag, `"${correctMD5}"`); - await getAwsError(key, 'NoSuchKey'); - }); - - it('should put actual object with body and mdonly header', async () => { - const key = `mdonly-${genUniqID()}`; - const b64 = Buffer.from(correctMD5, 'hex').toString('base64'); - const params = { Bucket: bucket, Key: key, Body: body, - Metadata: { 'scal-location-constraint': awsLocation, - 'mdonly': 'true', - 'md5chksum': b64, - 'size': body.length.toString(), - } }; - await s3.send(new PutObjectCommand(params)).then(() => { - process.stdout.write('Putting object succeeded\n'); - }).catch(err => { - throw new Error(`Expected success, got error: ${err}`); - }); - const res = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key })); - assert.strictEqual(res.ETag, `"${correctMD5}"`); - await awsGetCheck(key, correctMD5, correctMD5, awsLocation); - }); - - it('should put 0-byte normally with mdonly header', async () => { - const key = `mdonly-${genUniqID()}`; - const b64 = Buffer.from(emptyMD5, 'hex').toString('base64'); - const params = { Bucket: bucket, Key: key, - Metadata: { 'scal-location-constraint': awsLocation, - 'mdonly': 'true', - 'md5chksum': b64, - 'size': '0', - } }; - await s3.send(new PutObjectCommand(params)).then(() => { - process.stdout.write('Putting object succeeded\n'); - }).catch(err => { - throw new Error(`Expected success, got error: ${err}`); - }); - await awsGetCheck(key, emptyMD5, emptyMD5, awsLocation); - }); - - it('should put a 0-byte object to AWS', async () => { - const key = `somekey-${genUniqID()}`; - const params = { Bucket: bucket, Key: key, - Metadata: { 'scal-location-constraint': awsLocation }, - }; - await s3.send(new PutObjectCommand(params)).then(() => { - process.stdout.write('Putting object succeeded\n'); - }).catch(err => { - throw new Error(`Expected success, got error: ${err}`); + it('should return an error to put request without a valid ' + 'location constraint', async () => { + const key = `somekey-${genUniqID()}`; + const params = { + Bucket: bucket, + Key: key, + Body: body, + Metadata: { 'scal-location-constraint': 'fail-region' }, + }; + try { + await s3.send(new PutObjectCommand(params)); + throw new Error('Expected failure but got success'); + } catch (err) { + assert.strictEqual(err.code, 'InvalidArgument'); + } }); - await awsGetCheck(key, emptyMD5, emptyMD5, awsLocation); - }); - it('should put an object to file', async () => { - const key = `somekey-${genUniqID()}`; - const params = { Bucket: bucket, Key: key, - Body: body, - Metadata: { 'scal-location-constraint': fileLocation }, - }; - await s3.send(new PutObjectCommand(params)).then(() => { - process.stdout.write('Putting object succeeded\n'); - }).catch(err => { - throw new Error(`Expected success, got error: ${err}`); + it('should put an object to mem', async () => { + const key = `somekey-${genUniqID()}`; + const params = { + Bucket: bucket, + Key: key, + Body: body, + Metadata: { 'scal-location-constraint': memLocation }, + }; + + await s3 + .send(new PutObjectCommand(params)) + .then(() => { + process.stdout.write('Putting object succeeded\n'); + }) + .catch(err => { + throw new Error(`Expected success, got error: ${err}`); + }); + const res = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key })); + assert.strictEqual(res.ETag, `"${correctMD5}"`); }); - const res = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key })); - assert.strictEqual(res.ETag, `"${correctMD5}"`); - }); - it('should put an object to AWS', async () => { - const key = `somekey-${genUniqID()}`; - const params = { Bucket: bucket, Key: key, - Body: body, - Metadata: { 'scal-location-constraint': awsLocation } }; - await s3.send(new PutObjectCommand(params)).then(() => { - process.stdout.write('Putting object succeeded\n'); - }).catch(err => { - throw new Error(`Expected success, got error: ${err}`); + it('should put a 0-byte object to mem', async () => { + const key = `somekey-${genUniqID()}`; + const params = { Bucket: bucket, Key: key, Metadata: { 'scal-location-constraint': memLocation } }; + + await s3 + .send(new PutObjectCommand(params)) + .then(() => { + process.stdout.write('Putting object succeeded\n'); + }) + .catch(err => { + throw new Error(`Expected success, got error: ${err}`); + }); + const res = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key })); + assert.strictEqual(res.ETag, `"${emptyMD5}"`); }); - await awsGetCheck(key, correctMD5, correctMD5, awsLocation); - }); - - it('should encrypt body only if bucket encrypted putting ' + - 'object to AWS', - async () => { - const key = `somekey-${genUniqID()}`; - const params = { Bucket: bucket, Key: key, - Body: body, - Metadata: { 'scal-location-constraint': awsLocation } }; - await s3.send(new PutObjectCommand(params)).then(() => { - process.stdout.write('Putting object succeeded\n'); - }).catch(err => { - throw new Error(`Expected success, got error: ${err}`); + + it('should put only metadata to mem with mdonly header', async () => { + const key = `mdonly-${genUniqID()}`; + const b64 = Buffer.from(correctMD5, 'hex').toString('base64'); + const params = { + Bucket: bucket, + Key: key, + Metadata: { + 'scal-location-constraint': awsLocation, + mdonly: 'true', + md5chksum: b64, + size: body.length.toString(), + }, + }; + await s3 + .send(new PutObjectCommand(params)) + .then(() => { + process.stdout.write('Putting object succeeded\n'); + }) + .catch(err => { + throw new Error(`Expected success, got error: ${err}`); + }); + const res = await s3.send(new HeadObjectCommand({ Bucket: bucket, Key: key })); + assert.strictEqual(res.ETag, `"${correctMD5}"`); + await getAwsError(key, 'NoSuchKey'); }); - await getAwsSuccess(key, correctMD5, awsLocation); - }); + it('should put actual object with body and mdonly header', async () => { + const key = `mdonly-${genUniqID()}`; + const b64 = Buffer.from(correctMD5, 'hex').toString('base64'); + const params = { + Bucket: bucket, + Key: key, + Body: body, + Metadata: { + 'scal-location-constraint': awsLocation, + mdonly: 'true', + md5chksum: b64, + size: body.length.toString(), + }, + }; + await s3 + .send(new PutObjectCommand(params)) + .then(() => { + process.stdout.write('Putting object succeeded\n'); + }) + .catch(err => { + throw new Error(`Expected success, got error: ${err}`); + }); + const res = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key })); + assert.strictEqual(res.ETag, `"${correctMD5}"`); + await awsGetCheck(key, correctMD5, correctMD5, awsLocation); + }); - it('should put an object to AWS with encryption', async () => { - const key = `somekey-${genUniqID()}`; - const params = { Bucket: bucket, Key: key, - Body: body, - Metadata: { 'scal-location-constraint': - awsLocationEncryption } }; - await s3.send(new PutObjectCommand(params)).then(() => { - process.stdout.write('Putting object succeeded\n'); - }).catch(err => { - throw new Error(`Expected success, got error: ${err}`); + it('should put 0-byte normally with mdonly header', async () => { + const key = `mdonly-${genUniqID()}`; + const b64 = Buffer.from(emptyMD5, 'hex').toString('base64'); + const params = { + Bucket: bucket, + Key: key, + Metadata: { + 'scal-location-constraint': awsLocation, + mdonly: 'true', + md5chksum: b64, + size: '0', + }, + }; + await s3 + .send(new PutObjectCommand(params)) + .then(() => { + process.stdout.write('Putting object succeeded\n'); + }) + .catch(err => { + throw new Error(`Expected success, got error: ${err}`); + }); + await awsGetCheck(key, emptyMD5, emptyMD5, awsLocation); }); - await awsGetCheck(key, correctMD5, correctMD5, - awsLocationEncryption); - }); - - it('should return a version id putting object to ' + - 'to AWS with versioning enabled', async () => { - const key = `somekey-${genUniqID()}`; - const params = { Bucket: bucket, Key: key, Body: body, - Metadata: { 'scal-location-constraint': awsLocation } }; - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningEnabled, - })); - const res = await s3.send(new PutObjectCommand(params)); - assert.strictEqual(res.VersionId); - await getAwsSuccess(key, correctMD5, awsLocation); - }); - - it('should put a large object to AWS', async () => { - const key = `somekey-${genUniqID()}`; - const params = { Bucket: bucket, Key: key, - Body: bigBody, - Metadata: { 'scal-location-constraint': awsLocation } }; - await s3.send(new PutObjectCommand(params)).then(() => { - process.stdout.write('Putting object succeeded\n'); - }).catch(err => { - throw new Error(`Expected success, got error: ${err}`); + + it('should put a 0-byte object to AWS', async () => { + const key = `somekey-${genUniqID()}`; + const params = { Bucket: bucket, Key: key, Metadata: { 'scal-location-constraint': awsLocation } }; + await s3 + .send(new PutObjectCommand(params)) + .then(() => { + process.stdout.write('Putting object succeeded\n'); + }) + .catch(err => { + throw new Error(`Expected success, got error: ${err}`); + }); + await awsGetCheck(key, emptyMD5, emptyMD5, awsLocation); }); - await awsGetCheck(key, bigS3MD5, bigAWSMD5, awsLocation); - }); - it('should put objects with same key to AWS ' + - 'then file, and object should only be present in file', async () => { - const key = `somekey-${genUniqID()}`; - const params = { Bucket: bucket, Key: key, - Body: body, - Metadata: { 'scal-location-constraint': awsLocation } }; - await s3.send(new PutObjectCommand(params)).then(() => { - process.stdout.write('Putting object succeeded\n'); - }).catch(err => { - throw new Error(`Expected success, got error: ${err}`); + it('should put an object to file', async () => { + const key = `somekey-${genUniqID()}`; + const params = { + Bucket: bucket, + Key: key, + Body: body, + Metadata: { 'scal-location-constraint': fileLocation }, + }; + await s3 + .send(new PutObjectCommand(params)) + .then(() => { + process.stdout.write('Putting object succeeded\n'); + }) + .catch(err => { + throw new Error(`Expected success, got error: ${err}`); + }); + const res = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key })); + assert.strictEqual(res.ETag, `"${correctMD5}"`); }); - params.Metadata = - { 'scal-location-constraint': fileLocation }; - await s3.send(new PutObjectCommand(params)).then(() => { - process.stdout.write('Putting object succeeded\n'); - }).catch(err => { - throw new Error(`Expected success, got error: ${err}`); + + it('should put an object to AWS', async () => { + const key = `somekey-${genUniqID()}`; + const params = { + Bucket: bucket, + Key: key, + Body: body, + Metadata: { 'scal-location-constraint': awsLocation }, + }; + await s3 + .send(new PutObjectCommand(params)) + .then(() => { + process.stdout.write('Putting object succeeded\n'); + }) + .catch(err => { + throw new Error(`Expected success, got error: ${err}`); + }); + await awsGetCheck(key, correctMD5, correctMD5, awsLocation); }); - const res = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key })); - assert.strictEqual( - res.Metadata['scal-location-constraint'], - fileLocation); - return await getAwsError(key, 'NoSuchKey'); - }); - - it('should put objects with same key to file ' + - 'then AWS, and object should only be present on AWS', async () => { - const key = `somekey-${genUniqID()}`; - const params = { Bucket: bucket, Key: key, - Body: body, - Metadata: { 'scal-location-constraint': fileLocation } }; - await s3.send(new PutObjectCommand(params)).then(() => { - process.stdout.write('Putting object succeeded\n'); - }).catch(err => { - throw new Error(`Expected success, got error: ${err}`); + + it('should encrypt body only if bucket encrypted putting ' + 'object to AWS', async () => { + const key = `somekey-${genUniqID()}`; + const params = { + Bucket: bucket, + Key: key, + Body: body, + Metadata: { 'scal-location-constraint': awsLocation }, + }; + await s3 + .send(new PutObjectCommand(params)) + .then(() => { + process.stdout.write('Putting object succeeded\n'); + }) + .catch(err => { + throw new Error(`Expected success, got error: ${err}`); + }); + await getAwsSuccess(key, correctMD5, awsLocation); }); - params.Metadata = { - 'scal-location-constraint': awsLocation }; - await s3.send(new PutObjectCommand(params)).then(() => { - process.stdout.write('Putting object succeeded\n'); - }).catch(err => { - throw new Error(`Expected success, got error: ${err}`); + + it('should put an object to AWS with encryption', async () => { + const key = `somekey-${genUniqID()}`; + const params = { + Bucket: bucket, + Key: key, + Body: body, + Metadata: { 'scal-location-constraint': awsLocationEncryption }, + }; + await s3 + .send(new PutObjectCommand(params)) + .then(() => { + process.stdout.write('Putting object succeeded\n'); + }) + .catch(err => { + throw new Error(`Expected success, got error: ${err}`); + }); + await awsGetCheck(key, correctMD5, correctMD5, awsLocationEncryption); }); - await awsGetCheck(key, correctMD5, correctMD5, - awsLocation); - }); - - it('should put two objects to AWS with same ' + - 'key, and newest object should be returned', async () => { - const key = `somekey-${genUniqID()}`; - const params = { Bucket: bucket, Key: key, - Body: body, - Metadata: { 'scal-location-constraint': awsLocation, - 'unique-header': 'first object' } }; - await s3.send(new PutObjectCommand(params)).then(() => { - process.stdout.write('Putting object succeeded\n'); - }).catch(err => { - throw new Error(`Expected success, got error: ${err}`); + + it('should return a version id putting object to ' + 'to AWS with versioning enabled', async () => { + const key = `somekey-${genUniqID()}`; + const params = { + Bucket: bucket, + Key: key, + Body: body, + Metadata: { 'scal-location-constraint': awsLocation }, + }; + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningEnabled, + }), + ); + const res = await s3.send(new PutObjectCommand(params)); + assert.strictEqual(res.VersionId); + await getAwsSuccess(key, correctMD5, awsLocation); }); - params.Metadata = { 'scal-location-constraint': awsLocation, - 'unique-header': 'second object' }; - await s3.send(new PutObjectCommand(params)).then(() => { - process.stdout.write('Putting object succeeded\n'); - }).catch(err => { - throw new Error(`Expected success, got error: ${err}`); + + it('should put a large object to AWS', async () => { + const key = `somekey-${genUniqID()}`; + const params = { + Bucket: bucket, + Key: key, + Body: bigBody, + Metadata: { 'scal-location-constraint': awsLocation }, + }; + await s3 + .send(new PutObjectCommand(params)) + .then(() => { + process.stdout.write('Putting object succeeded\n'); + }) + .catch(err => { + throw new Error(`Expected success, got error: ${err}`); + }); + await awsGetCheck(key, bigS3MD5, bigAWSMD5, awsLocation); }); - await awsGetCheck(key, correctMD5, correctMD5, - awsLocation, result => { - assert.strictEqual(result.Metadata - ['unique-header'], 'second object'); - }); - }); - }); + + it( + 'should put objects with same key to AWS ' + 'then file, and object should only be present in file', + async () => { + const key = `somekey-${genUniqID()}`; + const params = { + Bucket: bucket, + Key: key, + Body: body, + Metadata: { 'scal-location-constraint': awsLocation }, + }; + await s3 + .send(new PutObjectCommand(params)) + .then(() => { + process.stdout.write('Putting object succeeded\n'); + }) + .catch(err => { + throw new Error(`Expected success, got error: ${err}`); + }); + params.Metadata = { 'scal-location-constraint': fileLocation }; + await s3 + .send(new PutObjectCommand(params)) + .then(() => { + process.stdout.write('Putting object succeeded\n'); + }) + .catch(err => { + throw new Error(`Expected success, got error: ${err}`); + }); + const res = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key })); + assert.strictEqual(res.Metadata['scal-location-constraint'], fileLocation); + return await getAwsError(key, 'NoSuchKey'); + }, + ); + + it( + 'should put objects with same key to file ' + 'then AWS, and object should only be present on AWS', + async () => { + const key = `somekey-${genUniqID()}`; + const params = { + Bucket: bucket, + Key: key, + Body: body, + Metadata: { 'scal-location-constraint': fileLocation }, + }; + await s3 + .send(new PutObjectCommand(params)) + .then(() => { + process.stdout.write('Putting object succeeded\n'); + }) + .catch(err => { + throw new Error(`Expected success, got error: ${err}`); + }); + params.Metadata = { + 'scal-location-constraint': awsLocation, + }; + await s3 + .send(new PutObjectCommand(params)) + .then(() => { + process.stdout.write('Putting object succeeded\n'); + }) + .catch(err => { + throw new Error(`Expected success, got error: ${err}`); + }); + await awsGetCheck(key, correctMD5, correctMD5, awsLocation); + }, + ); + + it( + 'should put two objects to AWS with same ' + 'key, and newest object should be returned', + async () => { + const key = `somekey-${genUniqID()}`; + const params = { + Bucket: bucket, + Key: key, + Body: body, + Metadata: { 'scal-location-constraint': awsLocation, 'unique-header': 'first object' }, + }; + await s3 + .send(new PutObjectCommand(params)) + .then(() => { + process.stdout.write('Putting object succeeded\n'); + }) + .catch(err => { + throw new Error(`Expected success, got error: ${err}`); + }); + params.Metadata = { 'scal-location-constraint': awsLocation, 'unique-header': 'second object' }; + await s3 + .send(new PutObjectCommand(params)) + .then(() => { + process.stdout.write('Putting object succeeded\n'); + }) + .catch(err => { + throw new Error(`Expected success, got error: ${err}`); + }); + await awsGetCheck(key, correctMD5, correctMD5, awsLocation, result => { + assert.strictEqual(result.Metadata['unique-header'], 'second object'); + }); + }, + ); + }, + ); }); }); -describeSkipIfNotMultiple('MultipleBackend put object based on bucket location', -() => { +describeSkipIfNotMultiple('MultipleBackend put object based on bucket location', () => { withV4(sigCfg => { beforeEach(() => { bucketUtil = new BucketUtility('default', sigCfg); @@ -406,71 +493,89 @@ describeSkipIfNotMultiple('MultipleBackend put object based on bucket location', afterEach(async () => { process.stdout.write('Emptying bucket\n'); - await bucketUtil.empty(bucket) - .then(() => { - process.stdout.write('Deleting bucket\n'); - return bucketUtil.deleteOne(bucket); - }) - .catch(err => { - process.stdout.write(`Error in afterEach: ${err}\n`); - throw err; - }); + await bucketUtil + .empty(bucket) + .then(() => { + process.stdout.write('Deleting bucket\n'); + return bucketUtil.deleteOne(bucket); + }) + .catch(err => { + process.stdout.write(`Error in afterEach: ${err}\n`); + throw err; + }); }); - it('should put an object to mem with no location header', - async () => { + it('should put an object to mem with no location header', async () => { process.stdout.write('Creating bucket\n'); - await s3.send(new CreateBucketCommand({ Bucket: bucket, - CreateBucketConfiguration: { - LocationConstraint: memLocation, - }, - })); + await s3.send( + new CreateBucketCommand({ + Bucket: bucket, + CreateBucketConfiguration: { + LocationConstraint: memLocation, + }, + }), + ); process.stdout.write('Putting object\n'); const key = `somekey-${genUniqID()}`; const params = { Bucket: bucket, Key: key, Body: body }; - await s3.send(new PutObjectCommand(params)).then(() => { - process.stdout.write('Putting object succeeded\n'); - }).catch(err => { - throw new Error(`Expected success, got error: ${err}`); - }); + await s3 + .send(new PutObjectCommand(params)) + .then(() => { + process.stdout.write('Putting object succeeded\n'); + }) + .catch(err => { + throw new Error(`Expected success, got error: ${err}`); + }); const res = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key })); assert.strictEqual(res.ETag, `"${correctMD5}"`); }); it('should put an object to file with no location header', async () => { process.stdout.write('Creating bucket\n'); - await s3.send(new CreateBucketCommand({ Bucket: bucket, - CreateBucketConfiguration: { - LocationConstraint: fileLocation, - }, - })); + await s3.send( + new CreateBucketCommand({ + Bucket: bucket, + CreateBucketConfiguration: { + LocationConstraint: fileLocation, + }, + }), + ); process.stdout.write('Putting object\n'); const key = `somekey-${genUniqID()}`; const params = { Bucket: bucket, Key: key, Body: body }; - await s3.send(new PutObjectCommand(params)).then(() => { - process.stdout.write('Putting object succeeded\n'); - }).catch(err => { - throw new Error(`Expected success, got error: ${err}`); - }); + await s3 + .send(new PutObjectCommand(params)) + .then(() => { + process.stdout.write('Putting object succeeded\n'); + }) + .catch(err => { + throw new Error(`Expected success, got error: ${err}`); + }); const res = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key })); assert.strictEqual(res.ETag, `"${correctMD5}"`); }); it('should put an object to AWS with no location header', async () => { process.stdout.write('Creating bucket\n'); - await s3.send(new CreateBucketCommand({ Bucket: bucket, - CreateBucketConfiguration: { - LocationConstraint: awsLocation, - }, - })); + await s3.send( + new CreateBucketCommand({ + Bucket: bucket, + CreateBucketConfiguration: { + LocationConstraint: awsLocation, + }, + }), + ); process.stdout.write('Putting object\n'); const key = `somekey-${genUniqID()}`; const params = { Bucket: bucket, Key: key, Body: body }; - await s3.send(new PutObjectCommand(params)).then(() => { - process.stdout.write('Putting object succeeded\n'); - }).catch(err => { - throw new Error(`Expected success, got error: ${err}`); - }); + await s3 + .send(new PutObjectCommand(params)) + .then(() => { + process.stdout.write('Putting object succeeded\n'); + }) + .catch(err => { + throw new Error(`Expected success, got error: ${err}`); + }); await awsGetCheck(key, correctMD5, correctMD5, undefined); }); }); @@ -484,42 +589,47 @@ describe('MultipleBackend put based on request endpoint', () => { }); after(() => { process.stdout.write('Emptying bucket\n'); - return bucketUtil.empty(bucket) - .then(() => { - process.stdout.write('Deleting bucket\n'); - return bucketUtil.deleteOne(bucket); - }) - .catch(err => { - process.stdout.write(`Error in after: ${err}\n`); - throw err; - }); + return bucketUtil + .empty(bucket) + .then(() => { + process.stdout.write('Deleting bucket\n'); + return bucketUtil.deleteOne(bucket); + }) + .catch(err => { + process.stdout.write(`Error in after: ${err}\n`); + throw err; + }); }); it('should create bucket in corresponding backend', async () => { process.stdout.write('Creating bucket'); - + // Create bucket using AWS SDK v3 await s3.send(new CreateBucketCommand({ Bucket: bucket })); - + const key = `somekey-${genUniqID()}`; - - await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: body - })); - const locationData = await s3.send(new GetBucketLocationCommand({ Bucket: bucket })); - const objectData = await s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key - })); + + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: body, + }), + ); + const locationData = await s3.send(new GetBucketLocationCommand({ Bucket: bucket })); + const objectData = await s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + }), + ); const host = s3.config.endpoint.hostname; let endpoint = config.restEndpoints[host]; // s3 returns '' for us-east-1 if (endpoint === 'us-east-1') { endpoint = ''; } - + assert.strictEqual(locationData.LocationConstraint, endpoint); assert.strictEqual(objectData.ETag, `"${correctMD5}"`); }); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/put/putAzure.js b/tests/functional/aws-node-sdk/test/multipleBackend/put/putAzure.js index 90b0beb2cf..02247fd608 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/put/putAzure.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/put/putAzure.js @@ -1,11 +1,13 @@ const assert = require('assert'); const async = require('async'); -const { CreateBucketCommand, +const { + CreateBucketCommand, PutObjectCommand, GetObjectCommand, CreateMultipartUploadCommand, AbortMultipartUploadCommand, - PutBucketVersioningCommand } = require('@aws-sdk/client-s3'); + PutBucketVersioningCommand, +} = require('@aws-sdk/client-s3'); const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); const { @@ -40,7 +42,10 @@ let bucketUtil; let s3; function azureGetCheck(objectKey, azureMD5, azureMetadata, cb) { - azureClient.getContainerClient(azureContainerName).getBlobClient(objectKey).getProperties() + azureClient + .getContainerClient(azureContainerName) + .getBlobClient(objectKey) + .getProperties() .then(res => { const resMD5 = convertMD5(res.contentSettings.contentMD5); assert.strictEqual(resMD5, azureMD5); @@ -50,8 +55,7 @@ function azureGetCheck(objectKey, azureMD5, azureMetadata, cb) { .catch(err => cb(err)); } -describeSkipIfNotMultiple('MultipleBackend put object to AZURE', function -describeF() { +describeSkipIfNotMultiple('MultipleBackend put object to AZURE', function describeF() { this.timeout(250000); withV4(sigCfg => { beforeEach(function beforeEachF() { @@ -62,76 +66,84 @@ describeF() { afterEach(() => { process.stdout.write('Emptying bucket\n'); - return bucketUtil.empty(azureContainerName) - .then(() => { - process.stdout.write('Deleting bucket\n'); - return bucketUtil.deleteOne(azureContainerName); - }) - .catch(err => { - process.stdout.write(`Error in afterEach: ${err}\n`); - throw err; - }); + return bucketUtil + .empty(azureContainerName) + .then(() => { + process.stdout.write('Deleting bucket\n'); + return bucketUtil.deleteOne(azureContainerName); + }) + .catch(err => { + process.stdout.write(`Error in afterEach: ${err}\n`); + throw err; + }); }); describe('with bucket location header', () => { - beforeEach(done => { - s3.send(new CreateBucketCommand({ - Bucket: azureContainerName, - CreateBucketConfiguration: { - LocationConstraint: azureLocation, - }, - })) - .then(() => done()) - .catch(done); - }); - - it('should return a NotImplemented error if try to put ' + - 'versioning to bucket with Azure location', done => { - const params = { - Bucket: azureContainerName, - VersioningConfiguration: { - Status: 'Enabled', - }, - }; - s3.send(new PutBucketVersioningCommand(params)) - .then(() => { - done(new Error('Expected NotImplemented error')); - }) - .catch(err => { - assert.strictEqual(err.name, 'NotImplemented'); - done(); - }); + beforeEach(done => { + s3.send( + new CreateBucketCommand({ + Bucket: azureContainerName, + CreateBucketConfiguration: { + LocationConstraint: azureLocation, + }, + }), + ) + .then(() => done()) + .catch(done); }); - it('should put an object to Azure, with no object location ' + - 'header, based on bucket location', function it(done) { - const params = { - Bucket: azureContainerName, - Key: this.test.keyName, - Body: normalBody, - }; - async.waterfall([ - next => { - s3.send(new PutObjectCommand(params)) - .then(() => setTimeout(() => next(), azureTimeout)) - .catch(next); - }, - next => azureGetCheck(this.test.keyName, normalMD5, {}, - next), - ], done); - }); + it( + 'should return a NotImplemented error if try to put ' + 'versioning to bucket with Azure location', + done => { + const params = { + Bucket: azureContainerName, + VersioningConfiguration: { + Status: 'Enabled', + }, + }; + s3.send(new PutBucketVersioningCommand(params)) + .then(() => { + done(new Error('Expected NotImplemented error')); + }) + .catch(err => { + assert.strictEqual(err.name, 'NotImplemented'); + done(); + }); + }, + ); + + it( + 'should put an object to Azure, with no object location ' + 'header, based on bucket location', + function it(done) { + const params = { + Bucket: azureContainerName, + Key: this.test.keyName, + Body: normalBody, + }; + async.waterfall( + [ + next => { + s3.send(new PutObjectCommand(params)) + .then(() => setTimeout(() => next(), azureTimeout)) + .catch(next); + }, + next => azureGetCheck(this.test.keyName, normalMD5, {}, next), + ], + done, + ); + }, + ); }); describe('with no bucket location header', () => { beforeEach(() => - s3.send(new CreateBucketCommand({ Bucket: azureContainerName })) - .catch(err => { + s3.send(new CreateBucketCommand({ Bucket: azureContainerName })).catch(err => { process.stdout.write(`Error creating bucket: ${err}\n`); throw err; - })); + }), + ); keys.forEach(key => { - it(`should put a ${key.describe} object to Azure`, - function itF(done) { + it(`should put a ${key.describe} object to Azure`, function itF(done) { const params = { Bucket: azureContainerName, Key: this.test.keyName, @@ -140,22 +152,20 @@ describeF() { }; s3.send(new PutObjectCommand(params)) .then(() => { - setTimeout(() => - azureGetCheck(this.test.keyName, - key.MD5, azureMetadata, - () => done()), azureTimeout); + setTimeout( + () => azureGetCheck(this.test.keyName, key.MD5, azureMetadata, () => done()), + azureTimeout, + ); }) .catch(done); }); }); - it('should put a object to Azure location with bucketMatch=false', - function itF(done) { + it('should put a object to Azure location with bucketMatch=false', function itF(done) { const params = { Bucket: azureContainerName, Key: this.test.keyName, - Metadata: { 'scal-location-constraint': - azureLocationMismatch }, + Metadata: { 'scal-location-constraint': azureLocationMismatch }, Body: normalBody, }; const azureMetadataMismatch = { @@ -165,17 +175,21 @@ describeF() { }; s3.send(new PutObjectCommand(params)) .then(() => { - setTimeout(() => - azureGetCheck( - `${azureContainerName}/${this.test.keyName}`, - normalMD5, azureMetadataMismatch, - () => done()), azureTimeout); + setTimeout( + () => + azureGetCheck( + `${azureContainerName}/${this.test.keyName}`, + normalMD5, + azureMetadataMismatch, + () => done(), + ), + azureTimeout, + ); }) .catch(done); }); - it('should return error ServiceUnavailable putting an invalid ' + - 'key name to Azure', done => { + it('should return error ServiceUnavailable putting an invalid ' + 'key name to Azure', done => { const params = { Bucket: azureContainerName, Key: '.', @@ -192,18 +206,20 @@ describeF() { }); }); - it('should return error NotImplemented putting a ' + - 'version to Azure', function itF(done) { - s3.send(new PutBucketVersioningCommand({ - Bucket: azureContainerName, - VersioningConfiguration: versioningEnabled, - })) + it('should return error NotImplemented putting a ' + 'version to Azure', function itF(done) { + s3.send( + new PutBucketVersioningCommand({ + Bucket: azureContainerName, + VersioningConfiguration: versioningEnabled, + }), + ) .then(() => { - const params = { Bucket: azureContainerName, + const params = { + Bucket: azureContainerName, Key: this.test.keyName, Body: normalBody, - Metadata: { 'scal-location-constraint': - azureLocation } }; + Metadata: { 'scal-location-constraint': azureLocation }, + }; return s3.send(new PutObjectCommand(params)); }) .then(() => { @@ -215,110 +231,128 @@ describeF() { }); }); - it('should put two objects to Azure with same ' + - 'key, and newest object should be returned', function itF(done) { - const params = { - Bucket: azureContainerName, - Key: this.test.keyName, - Metadata: { 'scal-location-constraint': azureLocation }, - }; - async.waterfall([ - next => { - s3.send(new PutObjectCommand(params)) - .then(() => next()) - .catch(next); - }, - next => { - params.Body = normalBody; - s3.send(new PutObjectCommand(params)) - .then(() => setTimeout(() => next(), azureTimeout)) - .catch(next); - }, - next => { - setTimeout(() => { - azureGetCheck(this.test.keyName, normalMD5, - azureMetadata, next); - }, azureTimeout); - }, - ], done); - }); + it( + 'should put two objects to Azure with same ' + 'key, and newest object should be returned', + function itF(done) { + const params = { + Bucket: azureContainerName, + Key: this.test.keyName, + Metadata: { 'scal-location-constraint': azureLocation }, + }; + async.waterfall( + [ + next => { + s3.send(new PutObjectCommand(params)) + .then(() => next()) + .catch(next); + }, + next => { + params.Body = normalBody; + s3.send(new PutObjectCommand(params)) + .then(() => setTimeout(() => next(), azureTimeout)) + .catch(next); + }, + next => { + setTimeout(() => { + azureGetCheck(this.test.keyName, normalMD5, azureMetadata, next); + }, azureTimeout); + }, + ], + done, + ); + }, + ); - it('should put objects with same key to Azure ' + - 'then file, and object should only be present in file', function - itF(done) { - const params = { - Bucket: azureContainerName, - Key: this.test.keyName, - Body: normalBody, - Metadata: { 'scal-location-constraint': azureLocation } }; - async.waterfall([ - next => { - s3.send(new PutObjectCommand(params)) - .then(() => next()) - .catch(next); - }, - next => { - params.Metadata = { 'scal-location-constraint': - fileLocation }; - s3.send(new PutObjectCommand(params)) - .then(() => setTimeout(() => next(), azureTimeout)) - .catch(next); - }, - next => { - s3.send(new GetObjectCommand({ - Bucket: azureContainerName, - Key: this.test.keyName, - })) - .then(res => { - assert.strictEqual( - res.Metadata['scal-location-constraint'], - fileLocation); - next(); - }) - .catch(next); - }, - next => { - azureClient.getContainerClient(azureContainerName) - .getBlobClient(this.test.keyName).getProperties() - .then(() => { - next(new Error('Expected NotFound error')); - }) - .catch(err => { - assert.strictEqual(err.name, 'NotFound'); - next(); - }); - }, - ], done); - }); + it( + 'should put objects with same key to Azure ' + 'then file, and object should only be present in file', + function itF(done) { + const params = { + Bucket: azureContainerName, + Key: this.test.keyName, + Body: normalBody, + Metadata: { 'scal-location-constraint': azureLocation }, + }; + async.waterfall( + [ + next => { + s3.send(new PutObjectCommand(params)) + .then(() => next()) + .catch(next); + }, + next => { + params.Metadata = { 'scal-location-constraint': fileLocation }; + s3.send(new PutObjectCommand(params)) + .then(() => setTimeout(() => next(), azureTimeout)) + .catch(next); + }, + next => { + s3.send( + new GetObjectCommand({ + Bucket: azureContainerName, + Key: this.test.keyName, + }), + ) + .then(res => { + assert.strictEqual(res.Metadata['scal-location-constraint'], fileLocation); + next(); + }) + .catch(next); + }, + next => { + azureClient + .getContainerClient(azureContainerName) + .getBlobClient(this.test.keyName) + .getProperties() + .then(() => { + next(new Error('Expected NotFound error')); + }) + .catch(err => { + assert.strictEqual(err.name, 'NotFound'); + next(); + }); + }, + ], + done, + ); + }, + ); - it('should put objects with same key to file ' + - 'then Azure, and object should only be present on Azure', - function itF(done) { - const params = { Bucket: azureContainerName, Key: - this.test.keyName, - Body: normalBody, - Metadata: { 'scal-location-constraint': fileLocation } }; - async.waterfall([ - next => s3.send(new PutObjectCommand(params)).then(() => next()), - next => { - params.Metadata = { - 'scal-location-constraint': azureLocation, - }; - s3.send(new PutObjectCommand(params)).then(() => setTimeout(() => - next(), azureTimeout)); - }, - next => azureGetCheck(this.test.keyName, normalMD5, - azureMetadata, next), - ], done); - }); + it( + 'should put objects with same key to file ' + 'then Azure, and object should only be present on Azure', + function itF(done) { + const params = { + Bucket: azureContainerName, + Key: this.test.keyName, + Body: normalBody, + Metadata: { 'scal-location-constraint': fileLocation }, + }; + async.waterfall( + [ + next => s3.send(new PutObjectCommand(params)).then(() => next()), + next => { + params.Metadata = { + 'scal-location-constraint': azureLocation, + }; + s3.send(new PutObjectCommand(params)).then(() => + setTimeout(() => next(), azureTimeout), + ); + }, + next => azureGetCheck(this.test.keyName, normalMD5, azureMetadata, next), + ], + done, + ); + }, + ); describe('with ongoing MPU with same key name', () => { beforeEach(function beFn(done) { - s3.send(new CreateMultipartUploadCommand({ - Bucket: azureContainerName, - Key: this.currentTest.keyName, - Metadata: { 'scal-location-constraint': azureLocation }, - })) + s3.send( + new CreateMultipartUploadCommand({ + Bucket: azureContainerName, + Key: this.currentTest.keyName, + Metadata: { 'scal-location-constraint': azureLocation }, + }), + ) .then(res => { this.currentTest.uploadId = res.UploadId; done(); @@ -327,11 +361,13 @@ describeF() { }); afterEach(function afFn(done) { - s3.send(new AbortMultipartUploadCommand({ - Bucket: azureContainerName, - Key: this.currentTest.keyName, - UploadId: this.currentTest.uploadId, - })) + s3.send( + new AbortMultipartUploadCommand({ + Bucket: azureContainerName, + Key: this.currentTest.keyName, + UploadId: this.currentTest.uploadId, + }), + ) .then(() => { done(); }) @@ -339,11 +375,13 @@ describeF() { }); it('should return ServiceUnavailable', function itFn(done) { - s3.send(new PutObjectCommand({ - Bucket: azureContainerName, - Key: this.test.keyName, - Metadata: { 'scal-location-constraint': azureLocation }, - })) + s3.send( + new PutObjectCommand({ + Bucket: azureContainerName, + Key: this.test.keyName, + Metadata: { 'scal-location-constraint': azureLocation }, + }), + ) .then(() => { done(new Error('Expected ServiceUnavailable error')); }) diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/put/putGcp.js b/tests/functional/aws-node-sdk/test/multipleBackend/put/putGcp.js index 922b28399e..bfc818cce6 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/put/putGcp.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/put/putGcp.js @@ -1,14 +1,9 @@ const assert = require('assert'); -const { - CreateBucketCommand, - PutObjectCommand, - GetObjectCommand, -} = require('@aws-sdk/client-s3'); +const { CreateBucketCommand, PutObjectCommand, GetObjectCommand } = require('@aws-sdk/client-s3'); const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); -const { gcpClient, gcpBucket, - gcpLocation, fileLocation, genUniqID, describeSkipIfNotMultiple } = require('../utils'); +const { gcpClient, gcpBucket, gcpLocation, fileLocation, genUniqID, describeSkipIfNotMultiple } = require('../utils'); const bucket = `putgcp${genUniqID()}`; const body = Buffer.from('I am a body', 'utf8'); @@ -25,35 +20,37 @@ const retryTimeout = 10000; const maxGetRetries = 3; function checkGcp(key, gcpMD5, location, callback) { - gcpClient.getObject({ - Bucket: gcpBucket, - Key: key, - }, (err, res) => { - assert.equal(err, null, `Expected success, got error ${err}`); - if (res.Metadata && res.Metadata['scal-etag']) { - assert.strictEqual(res.Metadata['scal-etag'], gcpMD5); - } else { - assert.strictEqual( - res.ETag.substring(1, res.ETag.length - 1), gcpMD5); - } - assert.strictEqual(res.Metadata['scal-location-constraint'], - location); - callback(res); - }); + gcpClient.getObject( + { + Bucket: gcpBucket, + Key: key, + }, + (err, res) => { + assert.equal(err, null, `Expected success, got error ${err}`); + if (res.Metadata && res.Metadata['scal-etag']) { + assert.strictEqual(res.Metadata['scal-etag'], gcpMD5); + } else { + assert.strictEqual(res.ETag.substring(1, res.ETag.length - 1), gcpMD5); + } + assert.strictEqual(res.Metadata['scal-location-constraint'], location); + callback(res); + }, + ); } function checkGcpError(key, expectedError, callback) { setTimeout(() => { - gcpClient.getObject({ - Bucket: gcpBucket, - Key: key, - }, err => { - assert.notStrictEqual(err, undefined, - 'Expected error but did not find one'); - assert.strictEqual(err.code, expectedError, - `Expected error code ${expectedError} but got ${err.code}`); - callback(); - }); + gcpClient.getObject( + { + Bucket: gcpBucket, + Key: key, + }, + err => { + assert.notStrictEqual(err, undefined, 'Expected error but did not find one'); + assert.strictEqual(err.code, expectedError, `Expected error code ${expectedError} but got ${err.code}`); + callback(); + }, + ); }, 1000); } @@ -65,8 +62,7 @@ function gcpGetCheck(objectKey, s3MD5, gcpMD5, location, callback) { s3.send(new GetObjectCommand(params)) .then(res => { assert.strictEqual(res.ETag, `"${s3MD5}"`); - const metadataLocation = res.Metadata && - res.Metadata['scal-location-constraint']; + const metadataLocation = res.Metadata && res.Metadata['scal-location-constraint']; assert.strictEqual(metadataLocation, location); process.stdout.write('Getting object from GCP\n'); checkGcp(objectKey, gcpMD5, location, callback); @@ -83,22 +79,19 @@ function gcpGetCheck(objectKey, s3MD5, gcpMD5, location, callback) { }, retryTimeout); return; } - assert.strictEqual(err, null, 'Expected success, got error ' + - `on call to GCP through S3: ${err}`); + assert.strictEqual(err, null, 'Expected success, got error ' + `on call to GCP through S3: ${err}`); }); } attempt(); } -describeSkipIfNotMultiple('MultipleBackend put object to GCP', function -describeFn() { +describeSkipIfNotMultiple('MultipleBackend put object to GCP', function describeFn() { this.timeout(250000); withV4(sigCfg => { beforeEach(() => { bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; - return s3.send(new CreateBucketCommand({ Bucket: bucket })) - .catch(err => { + return s3.send(new CreateBucketCommand({ Bucket: bucket })).catch(err => { process.stdout.write(`Error creating bucket: ${err}\n`); throw err; }); @@ -106,19 +99,19 @@ describeFn() { afterEach(() => { process.stdout.write('Emptying bucket\n'); - return bucketUtil.empty(bucket) - .then(() => { - process.stdout.write('Deleting bucket\n'); - return bucketUtil.deleteOne(bucket); - }) - .catch(err => { - process.stdout.write(`Error in afterEach: ${err}\n`); - throw err; - }); + return bucketUtil + .empty(bucket) + .then(() => { + process.stdout.write('Deleting bucket\n'); + return bucketUtil.deleteOne(bucket); + }) + .catch(err => { + process.stdout.write(`Error in afterEach: ${err}\n`); + throw err; + }); }); - describe('with set location from "x-amz-meta-scal-' + - 'location-constraint" header', function describe() { + describe('with set location from "x-amz-meta-scal-' + 'location-constraint" header', function describe() { if (!process.env.S3_END_TO_END) { this.retries(2); } @@ -145,13 +138,16 @@ describeFn() { const { s3MD5, gcpMD5 } = test.output; it(test.msg, done => { const key = `somekey-${genUniqID()}`; - const params = { Bucket: bucket, Key: key, Body, + const params = { + Bucket: bucket, + Key: key, + Body, Metadata: { 'scal-location-constraint': location }, }; - return s3.send(new PutObjectCommand(params)) + return s3 + .send(new PutObjectCommand(params)) .then(() => { - gcpGetCheck(key, s3MD5, gcpMD5, location, - () => done()); + gcpGetCheck(key, s3MD5, gcpMD5, location, () => done()); }) .catch(done); }); @@ -163,71 +159,82 @@ describeFn() { this.retries(2); } - it('should put objects with same key to GCP ' + - 'then file, and object should only be present in file', done => { - const key = `somekey-${genUniqID()}`; - const params = { Bucket: bucket, Key: key, - Body: body, - Metadata: { 'scal-location-constraint': gcpLocation } }; - return s3.send(new PutObjectCommand(params)) - .then(() => { - params.Metadata = - { 'scal-location-constraint': fileLocation }; - return s3.send(new PutObjectCommand(params)); - }) - .then(() => s3.send(new GetObjectCommand({ + it( + 'should put objects with same key to GCP ' + 'then file, and object should only be present in file', + done => { + const key = `somekey-${genUniqID()}`; + const params = { Bucket: bucket, Key: key, - }))) - .then(res => { - assert.strictEqual( - res.Metadata['scal-location-constraint'], - fileLocation); - checkGcpError(key, 'NoSuchKey', - () => done()); - }) - .catch(done); - }); + Body: body, + Metadata: { 'scal-location-constraint': gcpLocation }, + }; + return s3 + .send(new PutObjectCommand(params)) + .then(() => { + params.Metadata = { 'scal-location-constraint': fileLocation }; + return s3.send(new PutObjectCommand(params)); + }) + .then(() => + s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + }), + ), + ) + .then(res => { + assert.strictEqual(res.Metadata['scal-location-constraint'], fileLocation); + checkGcpError(key, 'NoSuchKey', () => done()); + }) + .catch(done); + }, + ); - it('should put objects with same key to file ' + - 'then GCP, and object should only be present on GCP', done => { - const key = `somekey-${genUniqID()}`; - const params = { Bucket: bucket, Key: key, - Body: body, - Metadata: { 'scal-location-constraint': fileLocation } }; - return s3.send(new PutObjectCommand(params)) - .then(() => { - params.Metadata = { - 'scal-location-constraint': gcpLocation }; - return s3.send(new PutObjectCommand(params)); - }) - .then(() => { - gcpGetCheck(key, correctMD5, correctMD5, - gcpLocation, () => done()); - }) - .catch(done); - }); + it( + 'should put objects with same key to file ' + 'then GCP, and object should only be present on GCP', + done => { + const key = `somekey-${genUniqID()}`; + const params = { + Bucket: bucket, + Key: key, + Body: body, + Metadata: { 'scal-location-constraint': fileLocation }, + }; + return s3 + .send(new PutObjectCommand(params)) + .then(() => { + params.Metadata = { + 'scal-location-constraint': gcpLocation, + }; + return s3.send(new PutObjectCommand(params)); + }) + .then(() => { + gcpGetCheck(key, correctMD5, correctMD5, gcpLocation, () => done()); + }) + .catch(done); + }, + ); - it('should put two objects to GCP with same ' + - 'key, and newest object should be returned', done => { + it('should put two objects to GCP with same ' + 'key, and newest object should be returned', done => { const key = `somekey-${genUniqID()}`; - const params = { Bucket: bucket, Key: key, + const params = { + Bucket: bucket, + Key: key, Body: body, - Metadata: { 'scal-location-constraint': gcpLocation, - 'unique-header': 'first object' } }; - return s3.send(new PutObjectCommand(params)) + Metadata: { 'scal-location-constraint': gcpLocation, 'unique-header': 'first object' }, + }; + return s3 + .send(new PutObjectCommand(params)) .then(() => { - params.Metadata = { 'scal-location-constraint': gcpLocation, - 'unique-header': 'second object' }; + params.Metadata = { 'scal-location-constraint': gcpLocation, 'unique-header': 'second object' }; return s3.send(new PutObjectCommand(params)); }) .then(() => { - gcpGetCheck(key, correctMD5, correctMD5, - gcpLocation, result => { - assert.strictEqual(result.Metadata - ['unique-header'], 'second object'); - done(); - }); + gcpGetCheck(key, correctMD5, correctMD5, gcpLocation, result => { + assert.strictEqual(result.Metadata['unique-header'], 'second object'); + done(); + }); }) .catch(done); }); @@ -244,33 +251,37 @@ describeSkipIfNotMultiple('MultipleBackend put object based on bucket location', afterEach(() => { process.stdout.write('Emptying bucket\n'); - return bucketUtil.empty(bucket) - .then(() => { - process.stdout.write('Deleting bucket\n'); - return bucketUtil.deleteOne(bucket); - }) - .catch(err => { - process.stdout.write(`Error in afterEach: ${err}\n`); - throw err; - }); + return bucketUtil + .empty(bucket) + .then(() => { + process.stdout.write('Deleting bucket\n'); + return bucketUtil.deleteOne(bucket); + }) + .catch(err => { + process.stdout.write(`Error in afterEach: ${err}\n`); + throw err; + }); }); it('should put an object to GCP with no location header', done => { process.stdout.write('Creating bucket\n'); const key = `somekey-${genUniqID()}`; const params = { Bucket: bucket, Key: key, Body: body }; - return s3.send(new CreateBucketCommand({ Bucket: bucket, - CreateBucketConfiguration: { - LocationConstraint: gcpLocation, - }, - })) + return s3 + .send( + new CreateBucketCommand({ + Bucket: bucket, + CreateBucketConfiguration: { + LocationConstraint: gcpLocation, + }, + }), + ) .then(() => { process.stdout.write('Putting object\n'); return s3.send(new PutObjectCommand(params)); }) .then(() => { - gcpGetCheck(key, correctMD5, correctMD5, undefined, - () => done()); + gcpGetCheck(key, correctMD5, correctMD5, undefined, () => done()); }) .catch(done); }); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/unknownEndpoint.js b/tests/functional/aws-node-sdk/test/multipleBackend/unknownEndpoint.js index 9b8cae4a8d..d0331d95b1 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/unknownEndpoint.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/unknownEndpoint.js @@ -32,39 +32,43 @@ describe('Requests to ip endpoint not in config', () => { await bucketUtil.deleteOne(bucket); }); - it('should accept put bucket request ' + - 'to IP address endpoint that is not in config using ' + - 'path style', + it( + 'should accept put bucket request ' + 'to IP address endpoint that is not in config using ' + 'path style', async () => { await s3.send(new CreateBucketCommand({ Bucket: bucket })); - }); + }, + ); const itSkipIfE2E = process.env.S3_END_TO_END ? it.skip : it; // skipping in E2E since in E2E 127.0.0.3 resolving to // localhost which is in config. Once integration is using // different machines we can update this. - itSkipIfE2E('should show us-east-1 as bucket location since' + - 'IP address endpoint was not in config thereby ' + - 'defaulting to us-east-1', + itSkipIfE2E( + 'should show us-east-1 as bucket location since' + + 'IP address endpoint was not in config thereby ' + + 'defaulting to us-east-1', async () => { const res = await s3.send(new GetBucketLocationCommand({ Bucket: bucket })); assert.strictEqual(res.LocationConstraint, undefined); - }); + }, + ); - it('should accept put object request ' + - 'to IP address endpoint that is not in config using ' + - 'path style and use the bucket location for the object', + it( + 'should accept put object request ' + + 'to IP address endpoint that is not in config using ' + + 'path style and use the bucket location for the object', async () => { await s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: body })); await s3.send(new HeadObjectCommand({ Bucket: bucket, Key: key })); - }); + }, + ); - it('should accept get object request ' + - 'to IP address endpoint that is not in config using ' + - 'path style', + it( + 'should accept get object request ' + 'to IP address endpoint that is not in config using ' + 'path style', async () => { const res = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key })); assert.strictEqual(res.ETag, expectedETag); - }); + }, + ); }); }); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/utils.js b/tests/functional/aws-node-sdk/test/multipleBackend/utils.js index e87fc5058b..8f8b31555f 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/utils.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/utils.js @@ -22,9 +22,7 @@ const { getRealAwsConfig } = require('../support/awsConfig'); const { config } = require('../../../../../lib/Config'); const authdata = require('../../../../../conf/authdata.json'); -const { - describeSkipIfNotMultiple, -} = require('../../lib/utility/test-utils'); +const { describeSkipIfNotMultiple } = require('../../lib/utility/test-utils'); const memLocation = 'scality-internal-mem'; const fileLocation = 'scality-internal-file'; @@ -56,23 +54,23 @@ if (config.backends.data === 'multiple') { awsS3 = new S3Client(awsConfig); awsBucket = config.locationConstraints[awsLocation].details.bucketName; } else { - process.stdout.write(`LocationConstraint for aws '${awsLocation}' not found in ${ - Object.keys(config.locationConstraints)}\n`); + process.stdout.write( + `LocationConstraint for aws '${awsLocation}' not found in ${Object.keys(config.locationConstraints)}\n`, + ); } if (config.locationConstraints[gcpLocation]) { const gcpConfig = getRealAwsConfig(gcpLocation); gcpClient = new GCP(gcpConfig); gcpBucket = config.locationConstraints[gcpLocation].details.bucketName; - gcpBucketMPU = - config.locationConstraints[gcpLocation].details.mpuBucketName; + gcpBucketMPU = config.locationConstraints[gcpLocation].details.mpuBucketName; } else { - process.stdout.write(`LocationConstraint for gcp '${gcpLocation}' not found in ${ - Object.keys(config.locationConstraints)}\n`); + process.stdout.write( + `LocationConstraint for gcp '${gcpLocation}' not found in ${Object.keys(config.locationConstraints)}\n`, + ); } } - const utils = { describeSkipIfNotMultiple, awsS3, @@ -137,11 +135,12 @@ utils.getAzureClient = () => { return true; } - if (config.locationConstraints[azureLocation] && + if ( + config.locationConstraints[azureLocation] && config.locationConstraints[azureLocation].details && - config.locationConstraints[azureLocation].details[key]) { - params[key] = - config.locationConstraints[azureLocation].details[key]; + config.locationConstraints[azureLocation].details[key] + ) { + params[key] = config.locationConstraints[azureLocation].details[key]; return true; } return false; @@ -151,20 +150,18 @@ utils.getAzureClient = () => { return undefined; } - const cred = new azure.StorageSharedKeyCredential( - params.azureStorageAccountName, - params.azureStorageAccessKey, - ); + const cred = new azure.StorageSharedKeyCredential(params.azureStorageAccountName, params.azureStorageAccessKey); return new azure.BlobServiceClient(params.azureStorageEndpoint, cred); }; utils.getAzureContainerName = azureLocation => { let azureContainerName; - if (config.locationConstraints[azureLocation] && - config.locationConstraints[azureLocation].details && - config.locationConstraints[azureLocation].details.azureContainerName) { - azureContainerName = - config.locationConstraints[azureLocation].details.azureContainerName; + if ( + config.locationConstraints[azureLocation] && + config.locationConstraints[azureLocation].details && + config.locationConstraints[azureLocation].details.azureContainerName + ) { + azureContainerName = config.locationConstraints[azureLocation].details.azureContainerName; } return azureContainerName; }; @@ -195,8 +192,7 @@ utils.getAzureKeys = () => { // For contentMD5, Azure requires base64 but AWS requires hex, so convert // from base64 to hex -utils.convertMD5 = contentMD5 => - Buffer.from(contentMD5, 'base64').toString('hex'); +utils.convertMD5 = contentMD5 => Buffer.from(contentMD5, 'base64').toString('hex'); utils.expectedETag = (body, getStringified = true) => { const eTagValue = crypto.createHash('md5').update(body).digest('hex'); @@ -215,9 +211,11 @@ utils.waitForVersioningBeforePut = async (s3, bucket, callback) => { for (let attempt = 1; attempt <= MAX_VERSIONING_CHECKS; attempt++) { let versioningEnabled = false; try { - const versioningResult = await s3.send(new GetBucketVersioningCommand({ - Bucket: bucket, - })); + const versioningResult = await s3.send( + new GetBucketVersioningCommand({ + Bucket: bucket, + }), + ); versioningEnabled = versioningResult.Status === 'Enabled'; } catch { if (attempt === MAX_VERSIONING_CHECKS) { @@ -244,34 +242,44 @@ utils.waitForVersioningBeforePut = async (s3, bucket, callback) => { }; utils.putToAwsBackend = (s3, bucket, key, body, callback) => { - const result = s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: body, - Metadata: { 'scal-location-constraint': awsLocation } - })); + const result = s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: body, + Metadata: { 'scal-location-constraint': awsLocation }, + }), + ); if (callback) { - return result.then(data => { - callback(null, data.VersionId); - }).catch(err => { - s3.send(new ListObjectVersionsCommand({ - Bucket: bucket, - Prefix: key, - })).then(data => { - callback(err, data.VersionId); - }).catch(listErr => { - callback(listErr); + return result + .then(data => { + callback(null, data.VersionId); + }) + .catch(err => { + s3.send( + new ListObjectVersionsCommand({ + Bucket: bucket, + Prefix: key, + }), + ) + .then(data => { + callback(err, data.VersionId); + }) + .catch(listErr => { + callback(listErr); + }); }); - }); } return result; }; utils.enableVersioning = (s3, bucket, callback) => { - const promise = s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningEnabled - })); + const promise = s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningEnabled, + }), + ); if (callback) { return promise.then(() => callback()).catch(err => callback(err)); @@ -280,11 +288,13 @@ utils.enableVersioning = (s3, bucket, callback) => { }; utils.suspendVersioning = (s3, bucket, callback) => { - const promise = s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningSuspended - })); - + const promise = s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningSuspended, + }), + ); + if (callback) { return promise.then(() => callback()).catch(err => callback(err)); } @@ -355,8 +365,7 @@ utils.putNullVersionsToAws = async (s3, bucket, key, versions, callback) => { utils.getAndAssertResult = (s3, params, callback) => { const run = async () => { - const { bucket, key, body, versionId, expectedVersionId, - expectedTagCount, expectedError } = params; + const { bucket, key, body, versionId, expectedVersionId, expectedTagCount, expectedError } = params; const getParams = { Bucket: bucket, Key: key, @@ -379,20 +388,25 @@ utils.getAndAssertResult = (s3, params, callback) => { chunks.push(chunk); } const bodyBuffer = Buffer.concat(chunks); - assert.equal(bodyBuffer.length, data.ContentLength, + assert.equal( + bodyBuffer.length, + data.ContentLength, `received data of length ${bodyBuffer.length} does not ` + - 'equal expected based on ' + - `content length header of ${data.ContentLength}`); + 'equal expected based on ' + + `content length header of ${data.ContentLength}`, + ); const expectedMD5 = utils.expectedETag(body, false); const resultMD5 = utils.expectedETag(bodyBuffer, false); assert.strictEqual(resultMD5, expectedMD5); } if (!expectedVersionId) { - assert.strictEqual(data.VersionId, undefined, - `Expected undefined VersionId but got ${data.VersionId}`); + assert.strictEqual(data.VersionId, undefined, `Expected undefined VersionId but got ${data.VersionId}`); } else { - assert.strictEqual(data.VersionId, expectedVersionId, - `Expected VersionId ${expectedVersionId} but got ${data.VersionId}`); + assert.strictEqual( + data.VersionId, + expectedVersionId, + `Expected VersionId ${expectedVersionId} but got ${data.VersionId}`, + ); } if (expectedTagCount && expectedTagCount === '0') { assert.strictEqual(data.TagCount, undefined); @@ -425,11 +439,11 @@ utils.getAwsRetry = (params, retryNumber, assertCb) => { }; const maxRetries = 2; const timeout = retryTimeout[retryNumber]; - + const executeGet = async () => { try { - const params = { - Bucket: awsBucket, + const params = { + Bucket: awsBucket, Key: key, VersionId: versionId, }; @@ -439,7 +453,7 @@ utils.getAwsRetry = (params, retryNumber, assertCb) => { return { success: false, error: err }; } }; - + return setTimeout(() => { executeGet() .then(result => { @@ -461,9 +475,8 @@ utils.getAwsRetry = (params, retryNumber, assertCb) => { utils.awsGetLatestVerId = (key, body, cb) => utils.getAwsRetry({ key }, 0, async (err, result) => { - assert.strictEqual(err, null, 'Expected success ' + - `getting object from AWS, got error ${err}`); - + assert.strictEqual(err, null, 'Expected success ' + `getting object from AWS, got error ${err}`); + const chunks = []; for await (const chunk of result.Body) { chunks.push(chunk); @@ -492,19 +505,21 @@ function _getTaggingConfig(tags) { utils.tagging.putTaggingAndAssert = async (s3, params) => { const { bucket, key, tags, versionId, expectedVersionId, expectedError } = params; const taggingConfig = _getTaggingConfig(tags); - + try { - const data = await s3.send(new PutObjectTaggingCommand({ - Bucket: bucket, - Key: key, - VersionId: versionId, - Tagging: taggingConfig - })); - + const data = await s3.send( + new PutObjectTaggingCommand({ + Bucket: bucket, + Key: key, + VersionId: versionId, + Tagging: taggingConfig, + }), + ); + if (expectedError) { throw new Error(`Expected error ${expectedError} but got success`); } - + if (expectedVersionId) { assert.strictEqual(data.VersionId, expectedVersionId); } else { @@ -521,35 +536,35 @@ utils.tagging.putTaggingAndAssert = async (s3, params) => { }; utils.tagging.getTaggingAndAssert = async (s3, params) => { - const { bucket, key, expectedTags, versionId, expectedVersionId, - expectedError, getObject } = params; - + const { bucket, key, expectedTags, versionId, expectedVersionId, expectedError, getObject } = params; + try { - const data = await s3.send(new GetObjectTaggingCommand({ - Bucket: bucket, - Key: key, - VersionId: versionId - })); - + const data = await s3.send( + new GetObjectTaggingCommand({ + Bucket: bucket, + Key: key, + VersionId: versionId, + }), + ); + if (expectedError) { throw new Error(`Expected error ${expectedError} but got success`); } - + const expectedTagResult = _getTaggingConfig(expectedTags); const expectedTagCount = `${Object.keys(expectedTags).length}`; - + if (expectedVersionId) { assert.strictEqual(data.VersionId, expectedVersionId); } else { assert.strictEqual(data.VersionId, undefined); } assert.deepStrictEqual(data.TagSet, expectedTagResult.TagSet); - + if (getObject !== false) { - await utils.getAndAssertResult(s3, { bucket, key, versionId, - expectedVersionId, expectedTagCount }); + await utils.getAndAssertResult(s3, { bucket, key, versionId, expectedVersionId, expectedTagCount }); } - + return data.VersionId; } catch (err) { if (expectedError) { @@ -562,26 +577,32 @@ utils.tagging.getTaggingAndAssert = async (s3, params) => { utils.tagging.delTaggingAndAssert = async (s3, params) => { const { bucket, key, versionId, expectedVersionId, expectedError } = params; - + try { - const data = await s3.send(new DeleteObjectTaggingCommand({ - Bucket: bucket, - Key: key, - VersionId: versionId - })); - + const data = await s3.send( + new DeleteObjectTaggingCommand({ + Bucket: bucket, + Key: key, + VersionId: versionId, + }), + ); + if (expectedError) { throw new Error(`Expected error ${expectedError} but got success`); } - + if (expectedVersionId) { assert.strictEqual(data.VersionId, expectedVersionId); } else { assert.strictEqual(data.VersionId, undefined); } - - await utils.tagging.getTaggingAndAssert(s3, { - bucket, key, versionId, expectedVersionId, expectedTags: {} + + await utils.tagging.getTaggingAndAssert(s3, { + bucket, + key, + versionId, + expectedVersionId, + expectedTags: {}, }); return undefined; } catch (err) { @@ -596,13 +617,15 @@ utils.tagging.delTaggingAndAssert = async (s3, params) => { utils.tagging.awsGetAssertTags = async params => { const { key, versionId, expectedTags } = params; const expectedTagResult = _getTaggingConfig(expectedTags); - - const data = await awsS3.send(new GetObjectTaggingCommand({ - Bucket: awsBucket, - Key: key, - VersionId: versionId - })); - + + const data = await awsS3.send( + new GetObjectTaggingCommand({ + Bucket: awsBucket, + Key: key, + VersionId: versionId, + }), + ); + assert.deepStrictEqual(data.TagSet, expectedTagResult.TagSet); }; diff --git a/tests/functional/aws-node-sdk/test/object/100-continue.js b/tests/functional/aws-node-sdk/test/object/100-continue.js index 9a41210f92..e43b40ba50 100644 --- a/tests/functional/aws-node-sdk/test/object/100-continue.js +++ b/tests/functional/aws-node-sdk/test/object/100-continue.js @@ -45,7 +45,7 @@ class ContinueRequestHandler { method: 'PUT', headers: { 'content-length': body.length, - 'Expect': this.expectHeader, + Expect: this.expectHeader, }, }; } @@ -102,8 +102,7 @@ class ContinueRequestHandler { req.flushHeaders(); // At this point we have only sent the header. const headerLen = req._header.length; - req.on('continue', () => - cb('Continue beeing seen when 403 is expected')); + req.on('continue', () => cb('Continue beeing seen when 403 is expected')); req.on('response', res => { res.on('data', () => {}); res.on('end', () => { @@ -138,30 +137,22 @@ describeSkipIfE2E('PUT public object with 100-continue header', () => { await s3.send(new CreateBucketCommand({ Bucket: bucket })); }); - afterEach(() => - bucketUtil.empty(bucket) - .then(() => bucketUtil.deleteOne(bucket))); + afterEach(() => bucketUtil.empty(bucket).then(() => bucketUtil.deleteOne(bucket))); - it('should return 200 status code', done => - continueRequest.hasStatusCode(200, done)); + it('should return 200 status code', done => continueRequest.hasStatusCode(200, done)); it('should return 200 status code with upper case value', done => - continueRequest.setExpectHeader('100-CONTINUE') - .hasStatusCode(200, done)); + continueRequest.setExpectHeader('100-CONTINUE').hasStatusCode(200, done)); it('should return 200 status code if incorrect value', done => - continueRequest.setExpectHeader('101-continue') - .hasStatusCode(200, done)); + continueRequest.setExpectHeader('101-continue').hasStatusCode(200, done)); it('should return 403 status code if cannot authenticate', done => - continueRequest.setRequestPath(invalidSignedURL) - .hasStatusCode(403, done)); + continueRequest.setRequestPath(invalidSignedURL).hasStatusCode(403, done)); - it('should wait for continue event before sending body', done => - continueRequest.sendsBodyOnContinue(done)); + it('should wait for continue event before sending body', done => continueRequest.sendsBodyOnContinue(done)); it('should not send continue if denied for a public user', done => - continueRequest.setRequestPath(invalidSignedURL) - .shouldNotGetContinue(done)); + continueRequest.setRequestPath(invalidSignedURL).shouldNotGetContinue(done)); }); }); diff --git a/tests/functional/aws-node-sdk/test/object/abortMPU.js b/tests/functional/aws-node-sdk/test/object/abortMPU.js index 7c3a836548..6db6283340 100644 --- a/tests/functional/aws-node-sdk/test/object/abortMPU.js +++ b/tests/functional/aws-node-sdk/test/object/abortMPU.js @@ -8,7 +8,7 @@ const async = require('async'); const { initMetadata, getMetadata } = require('../utils/init'); const metadata = require('../../../../../lib/metadata/wrapper'); const { DummyRequestLogger } = require('../../../../unit/helpers'); -const { +const { CreateBucketCommand, CreateMultipartUploadCommand, UploadPartCommand, @@ -20,7 +20,7 @@ const { DeleteObjectCommand, PutBucketVersioningCommand, HeadObjectCommand, - PutObjectCommand + PutObjectCommand, } = require('@aws-sdk/client-s3'); const date = Date.now(); @@ -38,17 +38,23 @@ async function cleanupVersionedBucket(bucketUtil, bucketName) { // Clean up all multipart uploads const listMPUResponse = await bucketUtil.s3.send(new ListMultipartUploadsCommand({ Bucket: bucketName })); if (listMPUResponse.Uploads && listMPUResponse.Uploads.length > 0) { - await Promise.all(listMPUResponse.Uploads.map(async upload => { - bucketUtil.s3.send(new AbortMultipartUploadCommand({ - Bucket: bucketName, - Key: upload.Key, - UploadId: upload.UploadId, - })).catch(err => { - if (err.name !== 'NoSuchUpload') { - throw err; - } - }); - })); + await Promise.all( + listMPUResponse.Uploads.map(async upload => { + bucketUtil.s3 + .send( + new AbortMultipartUploadCommand({ + Bucket: bucketName, + Key: upload.Key, + UploadId: upload.UploadId, + }), + ) + .catch(err => { + if (err.name !== 'NoSuchUpload') { + throw err; + } + }); + }), + ); } // Clean up all object versions @@ -67,15 +73,22 @@ describe('Abort MPU', () => { s3 = bucketUtil.s3; try { await s3.send(new CreateBucketCommand({ Bucket: bucket })); - const mpu = await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: key, - })); + const mpu = await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + }), + ); uploadId = mpu.UploadId; - await s3.send(new UploadPartCommand({ - Bucket: bucket, Key: key, - PartNumber: 1, UploadId: uploadId, Body: bodyFirstPart, - })); + await s3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: key, + PartNumber: 1, + UploadId: uploadId, + Body: bodyFirstPart, + }), + ); } catch (err) { process.stdout.write(`Error in beforeEach: ${err}\n`); throw err; @@ -83,32 +96,35 @@ describe('Abort MPU', () => { }); afterEach(async () => { - await s3.send(new AbortMultipartUploadCommand({ - Bucket: bucket, - Key: key, - UploadId: uploadId, - })); + await s3.send( + new AbortMultipartUploadCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + }), + ); await bucketUtil.empty(bucket); await bucketUtil.deleteOne(bucket); }); // aws-sdk now (v2.363.0) returns 'UriParameterError' error // this test was not replaced in any other suite - it.skip('should return InvalidRequest error if aborting without key', - done => { - s3.send(new AbortMultipartUploadCommand({ + it.skip('should return InvalidRequest error if aborting without key', done => { + s3.send( + new AbortMultipartUploadCommand({ Bucket: bucket, Key: '', - UploadId: uploadId - })) - .then(() => { - done(new Error('Expected failure but got success')); - }) - .catch(err => { - checkError(err, 'InvalidRequest', 'A key must be specified'); - done(); - }); - }); + UploadId: uploadId, + }), + ) + .then(() => { + done(new Error('Expected failure but got success')); + }) + .catch(err => { + checkError(err, 'InvalidRequest', 'A key must be specified'); + done(); + }); + }); }); }); @@ -137,86 +153,105 @@ describe('Abort MPU with existing object', function AbortMPUExistingObject() { let uploadId1; let uploadId2; let etag1; - async.waterfall([ - next => { - s3.send(new CreateMultipartUploadCommand({ - Bucket: bucketName, - Key: objectKey, - })) - .then(data => { - uploadId1 = data.UploadId; - return s3.send(new UploadPartCommand({ + async.waterfall( + [ + next => { + s3.send( + new CreateMultipartUploadCommand({ Bucket: bucketName, Key: objectKey, - PartNumber: 1, - UploadId: uploadId1, - Body: part1, - })); - }) - .then(data => { - etag1 = data.ETag; - return s3.send(new CompleteMultipartUploadCommand({ + }), + ) + .then(data => { + uploadId1 = data.UploadId; + return s3.send( + new UploadPartCommand({ + Bucket: bucketName, + Key: objectKey, + PartNumber: 1, + UploadId: uploadId1, + Body: part1, + }), + ); + }) + .then(data => { + etag1 = data.ETag; + return s3.send( + new CompleteMultipartUploadCommand({ + Bucket: bucketName, + Key: objectKey, + UploadId: uploadId1, + MultipartUpload: { Parts: [{ ETag: etag1, PartNumber: 1 }] }, + }), + ); + }) + .then(() => next()) + .catch(err => next(err)); + }, + next => { + s3.send( + new GetObjectCommand({ Bucket: bucketName, Key: objectKey, - UploadId: uploadId1, - MultipartUpload: { Parts: [{ ETag: etag1, PartNumber: 1 }] }, - })); - }) - .then(() => next()) - .catch(err => next(err)); - }, - next => { - s3.send(new GetObjectCommand({ - Bucket: bucketName, - Key: objectKey, - })) - .then(async data => { - const bodyText = await data.Body.transformToString(); - assert.strictEqual(bodyText, part1.toString()); - next(); - }) - .catch(err => next(err)); - }, - next => { - s3.send(new CreateMultipartUploadCommand({ - Bucket: bucketName, - Key: objectKey, - })) - .then(data => { - uploadId2 = data.UploadId; - return s3.send(new UploadPartCommand({ + }), + ) + .then(async data => { + const bodyText = await data.Body.transformToString(); + assert.strictEqual(bodyText, part1.toString()); + next(); + }) + .catch(err => next(err)); + }, + next => { + s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucketName, + Key: objectKey, + }), + ) + .then(data => { + uploadId2 = data.UploadId; + return s3.send( + new UploadPartCommand({ + Bucket: bucketName, + Key: objectKey, + PartNumber: 1, + UploadId: uploadId2, + Body: part2, + }), + ); + }) + .then(() => next()) + .catch(err => next(err)); + }, + next => { + s3.send( + new AbortMultipartUploadCommand({ Bucket: bucketName, Key: objectKey, - PartNumber: 1, UploadId: uploadId2, - Body: part2, - })); - }) - .then(() => next()) - .catch(err => next(err)); - }, - next => { - s3.send(new AbortMultipartUploadCommand({ - Bucket: bucketName, - Key: objectKey, - UploadId: uploadId2, - })) - .then(() => next()) - .catch(err => next(err)); - }, - next => { - s3.send(new GetObjectCommand({ - Bucket: bucketName, - Key: objectKey, - })) - .then(async data => { - const bodyText = await data.Body.transformToString(); - assert.strictEqual(bodyText, part1.toString()); - next(); - }) - .catch(err => next(err)); - }, - ], done); + }), + ) + .then(() => next()) + .catch(err => next(err)); + }, + next => { + s3.send( + new GetObjectCommand({ + Bucket: bucketName, + Key: objectKey, + }), + ) + .then(async data => { + const bodyText = await data.Body.transformToString(); + assert.strictEqual(bodyText, part1.toString()); + next(); + }) + .catch(err => next(err)); + }, + ], + done, + ); }); it('should not delete existing object data when aborting an old MPU for same key', done => { @@ -225,86 +260,105 @@ describe('Abort MPU with existing object', function AbortMPUExistingObject() { let uploadId1; let uploadId2; let etag2; - async.waterfall([ - next => { - s3.send(new CreateMultipartUploadCommand({ - Bucket: bucketName, - Key: objectKey, - })) - .then(data => { - uploadId1 = data.UploadId; - return s3.send(new UploadPartCommand({ + async.waterfall( + [ + next => { + s3.send( + new CreateMultipartUploadCommand({ Bucket: bucketName, Key: objectKey, - PartNumber: 1, - UploadId: uploadId1, - Body: part1, - })); - }) - .then(() => next()) - .catch(err => next(err)); - }, - next => { - s3.send(new CreateMultipartUploadCommand({ - Bucket: bucketName, - Key: objectKey, - })) - .then(data => { - uploadId2 = data.UploadId; - return s3.send(new UploadPartCommand({ + }), + ) + .then(data => { + uploadId1 = data.UploadId; + return s3.send( + new UploadPartCommand({ + Bucket: bucketName, + Key: objectKey, + PartNumber: 1, + UploadId: uploadId1, + Body: part1, + }), + ); + }) + .then(() => next()) + .catch(err => next(err)); + }, + next => { + s3.send( + new CreateMultipartUploadCommand({ Bucket: bucketName, Key: objectKey, - PartNumber: 1, - UploadId: uploadId2, - Body: part2, - })); - }) - .then(data => { - etag2 = data.ETag; - return s3.send(new CompleteMultipartUploadCommand({ + }), + ) + .then(data => { + uploadId2 = data.UploadId; + return s3.send( + new UploadPartCommand({ + Bucket: bucketName, + Key: objectKey, + PartNumber: 1, + UploadId: uploadId2, + Body: part2, + }), + ); + }) + .then(data => { + etag2 = data.ETag; + return s3.send( + new CompleteMultipartUploadCommand({ + Bucket: bucketName, + Key: objectKey, + UploadId: uploadId2, + MultipartUpload: { Parts: [{ ETag: etag2, PartNumber: 1 }] }, + }), + ); + }) + .then(() => next()) + .catch(err => next(err)); + }, + next => { + s3.send( + new GetObjectCommand({ Bucket: bucketName, Key: objectKey, - UploadId: uploadId2, - MultipartUpload: { Parts: [{ ETag: etag2, PartNumber: 1 }] }, - })); - }) - .then(() => next()) - .catch(err => next(err)); - }, - next => { - s3.send(new GetObjectCommand({ - Bucket: bucketName, - Key: objectKey, - })) - .then(async data => { - const bodyText = await data.Body.transformToString(); - assert.strictEqual(bodyText, part2.toString()); - next(); - }) - .catch(err => next(err)); - }, - next => { - s3.send(new AbortMultipartUploadCommand({ - Bucket: bucketName, - Key: objectKey, - UploadId: uploadId1, - })) - .then(() => next()) - .catch(err => next(err)); - }, - next => { - s3.send(new GetObjectCommand({ - Bucket: bucketName, - Key: objectKey, - })) - .then(async data => { - const bodyText = await data.Body.transformToString(); - assert.strictEqual(bodyText, part2.toString()); - next(); - }) - .catch(err => next(err)); - }, - ], done); + }), + ) + .then(async data => { + const bodyText = await data.Body.transformToString(); + assert.strictEqual(bodyText, part2.toString()); + next(); + }) + .catch(err => next(err)); + }, + next => { + s3.send( + new AbortMultipartUploadCommand({ + Bucket: bucketName, + Key: objectKey, + UploadId: uploadId1, + }), + ) + .then(() => next()) + .catch(err => next(err)); + }, + next => { + s3.send( + new GetObjectCommand({ + Bucket: bucketName, + Key: objectKey, + }), + ) + .then(async data => { + const bodyText = await data.Body.transformToString(); + assert.strictEqual(bodyText, part2.toString()); + next(); + }) + .catch(err => next(err)); + }, + ], + done, + ); }); }); }); @@ -322,22 +376,23 @@ describe('Abort MPU - No Such Upload', () => { afterEach(() => bucketUtil.deleteOne(bucket)); - it('should return NoSuchUpload error when aborting non-existent mpu', - done => { - s3.send(new AbortMultipartUploadCommand({ + it('should return NoSuchUpload error when aborting non-existent mpu', done => { + s3.send( + new AbortMultipartUploadCommand({ Bucket: bucket, Key: key, - UploadId: uuidv4().replace(/-/g, '') - })) - .then(() => { - done(new Error('Expected failure but got success')); - }) - .catch(err => { - assert.notEqual(err, null, 'Expected failure but got success'); - assert.strictEqual(err.name, 'NoSuchUpload'); - done(); - }); - }); + UploadId: uuidv4().replace(/-/g, ''), + }), + ) + .then(() => { + done(new Error('Expected failure but got success')); + }) + .catch(err => { + assert.notEqual(err, null, 'Expected failure but got success'); + assert.strictEqual(err.name, 'NoSuchUpload'); + done(); + }); + }); }); }); @@ -355,10 +410,12 @@ describe('Abort MPU - Versioned Bucket Cleanup', function testSuite() { s3 = bucketUtil.s3; await s3.send(new CreateBucketCommand({ Bucket: bucketName })); - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: { Status: 'Enabled' }, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ); }); afterEach(async () => { @@ -377,68 +434,83 @@ describe('Abort MPU - Versioned Bucket Cleanup', function testSuite() { currentVersion++; const data = Buffer.from(`Version ${currentVersion} data`); - async.waterfall([ - next => { - s3.send(new CreateMultipartUploadCommand({ - Bucket: bucketName, - Key: objectKey, - })) - .then(result => { - if (currentVersion === numberOfVersions) { - finalUploadId = result.UploadId; // Save the last one for aborting - } - next(null, result.UploadId); - }) - .catch(err => next(err)); - }, - (uploadId, next) => { - s3.send(new UploadPartCommand({ - Bucket: bucketName, - Key: objectKey, - PartNumber: 1, - UploadId: uploadId, - Body: data, - })) - .then(result => next(null, uploadId, result.ETag)) - .catch(err => next(err)); - }, - (uploadId, etag, next) => { - if (currentVersion === numberOfVersions) { - // Don't complete the last one - we'll abort it - return next(); - } - - return s3.send(new CompleteMultipartUploadCommand({ - Bucket: bucketName, - Key: objectKey, - UploadId: uploadId, - MultipartUpload: { - Parts: [{ ETag: etag, PartNumber: 1 }], - }, - })) - .then(() => next()) - .catch(err => next(err)); - }, - ], callback); + async.waterfall( + [ + next => { + s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucketName, + Key: objectKey, + }), + ) + .then(result => { + if (currentVersion === numberOfVersions) { + finalUploadId = result.UploadId; // Save the last one for aborting + } + next(null, result.UploadId); + }) + .catch(err => next(err)); + }, + (uploadId, next) => { + s3.send( + new UploadPartCommand({ + Bucket: bucketName, + Key: objectKey, + PartNumber: 1, + UploadId: uploadId, + Body: data, + }), + ) + .then(result => next(null, uploadId, result.ETag)) + .catch(err => next(err)); + }, + (uploadId, etag, next) => { + if (currentVersion === numberOfVersions) { + // Don't complete the last one - we'll abort it + return next(); + } + + return s3 + .send( + new CompleteMultipartUploadCommand({ + Bucket: bucketName, + Key: objectKey, + UploadId: uploadId, + MultipartUpload: { + Parts: [{ ETag: etag, PartNumber: 1 }], + }, + }), + ) + .then(() => next()) + .catch(err => next(err)); + }, + ], + callback, + ); }, err => { assert.ifError(err); // Now abort the final MPU - s3.send(new AbortMultipartUploadCommand({ - Bucket: bucketName, - Key: objectKey, - UploadId: finalUploadId, - })) + s3.send( + new AbortMultipartUploadCommand({ + Bucket: bucketName, + Key: objectKey, + UploadId: finalUploadId, + }), + ) .then(() => s3.send(new ListObjectVersionsCommand({ Bucket: bucketName }))) .then(data => { const objectVersions = data.Versions.filter(v => v.Key === objectKey); - assert.strictEqual(objectVersions.length, numberOfVersions - 1, - `Expected ${numberOfVersions - 1} versions after abort, got ${objectVersions.length}`); + assert.strictEqual( + objectVersions.length, + numberOfVersions - 1, + `Expected ${numberOfVersions - 1} versions after abort, got ${objectVersions.length}`, + ); done(); }) .catch(err => done(err)); - } + }, ); }); @@ -446,69 +518,83 @@ describe('Abort MPU - Versioned Bucket Cleanup', function testSuite() { let uploadId; const data = Buffer.from('test data for single MPU abort'); - async.waterfall([ - // Create and upload part for MPU - next => { - s3.send(new CreateMultipartUploadCommand({ - Bucket: bucketName, - Key: objectKey, - })) - .then(result => { - uploadId = result.UploadId; - next(); - }) - .catch(err => next(err)); - }, - next => { - s3.send(new UploadPartCommand({ - Bucket: bucketName, - Key: objectKey, - PartNumber: 1, - UploadId: uploadId, - Body: data, - })) - .then(() => next()) - .catch(err => next(err)); - }, + async.waterfall( + [ + // Create and upload part for MPU + next => { + s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucketName, + Key: objectKey, + }), + ) + .then(result => { + uploadId = result.UploadId; + next(); + }) + .catch(err => next(err)); + }, + next => { + s3.send( + new UploadPartCommand({ + Bucket: bucketName, + Key: objectKey, + PartNumber: 1, + UploadId: uploadId, + Body: data, + }), + ) + .then(() => next()) + .catch(err => next(err)); + }, - // Abort the MPU - next => { - s3.send(new AbortMultipartUploadCommand({ - Bucket: bucketName, - Key: objectKey, - UploadId: uploadId, - })) - .then(() => next()) - .catch(err => next(err)); - }, + // Abort the MPU + next => { + s3.send( + new AbortMultipartUploadCommand({ + Bucket: bucketName, + Key: objectKey, + UploadId: uploadId, + }), + ) + .then(() => next()) + .catch(err => next(err)); + }, - // Verify no object exists - next => { - s3.send(new GetObjectCommand({ - Bucket: bucketName, - Key: objectKey, - })) - .then(() => { - next(new Error('Expected NoSuchKey error')); - }) - .catch(err => { - assert.strictEqual(err.name, 'NoSuchKey'); - next(); - }); - }, + // Verify no object exists + next => { + s3.send( + new GetObjectCommand({ + Bucket: bucketName, + Key: objectKey, + }), + ) + .then(() => { + next(new Error('Expected NoSuchKey error')); + }) + .catch(err => { + assert.strictEqual(err.name, 'NoSuchKey'); + next(); + }); + }, - // Verify no versions exist - next => { - s3.send(new ListObjectVersionsCommand({ Bucket: bucketName })) - .then(data => { - const objectVersions = (data.Versions || []).filter(v => v.Key === objectKey); - assert.strictEqual(objectVersions.length, 0, - `Expected 0 versions after abort, got ${objectVersions.length}`); - next(); - }) - .catch(err => next(err)); - }, - ], done); + // Verify no versions exist + next => { + s3.send(new ListObjectVersionsCommand({ Bucket: bucketName })) + .then(data => { + const objectVersions = (data.Versions || []).filter(v => v.Key === objectKey); + assert.strictEqual( + objectVersions.length, + 0, + `Expected 0 versions after abort, got ${objectVersions.length}`, + ); + next(); + }) + .catch(err => next(err)); + }, + ], + done, + ); }); }); }); @@ -531,32 +617,44 @@ describe('Abort MPU - Orphan Cleanup', function testSuite() { * @param {boolean} isVersioned - Whether to create versioned metadata * @returns {Promise} Promise that resolves when orphaned metadata is created */ - async function createOrphanedObjectMetadata(s3Client, bucketName, objectKey, uploadIdToSimulate, - data, isVersioned) { + async function createOrphanedObjectMetadata( + s3Client, + bucketName, + objectKey, + uploadIdToSimulate, + data, + isVersioned, + ) { const tempObjectKey = `temp-object-for-metadata-${Date.now()}`; // Create temporary MPU and complete it to get real object metadata - const createResult = await s3Client.send(new CreateMultipartUploadCommand({ - Bucket: bucketName, - Key: tempObjectKey, - })); + const createResult = await s3Client.send( + new CreateMultipartUploadCommand({ + Bucket: bucketName, + Key: tempObjectKey, + }), + ); const tempUploadId = createResult.UploadId; - const uploadResult = await s3Client.send(new UploadPartCommand({ - Bucket: bucketName, - Key: tempObjectKey, - PartNumber: 1, - UploadId: tempUploadId, - Body: data, - })); + const uploadResult = await s3Client.send( + new UploadPartCommand({ + Bucket: bucketName, + Key: tempObjectKey, + PartNumber: 1, + UploadId: tempUploadId, + Body: data, + }), + ); const tempEtag = uploadResult.ETag; - const completeResult = await s3Client.send(new CompleteMultipartUploadCommand({ - Bucket: bucketName, - Key: tempObjectKey, - UploadId: tempUploadId, - MultipartUpload: { Parts: [{ ETag: tempEtag, PartNumber: 1 }] }, - })); + const completeResult = await s3Client.send( + new CompleteMultipartUploadCommand({ + Bucket: bucketName, + Key: tempObjectKey, + UploadId: tempUploadId, + MultipartUpload: { Parts: [{ ETag: tempEtag, PartNumber: 1 }] }, + }), + ); let tempVersionId; if (isVersioned && completeResult.VersionId) { @@ -569,14 +667,15 @@ describe('Abort MPU - Orphan Cleanup', function testSuite() { // Create a copy and override uploadId to match our test MPU // (simulating orphaned object) - const orphanedObjectMD = Object.assign({}, objMD, + const orphanedObjectMD = Object.assign( + {}, + objMD, // let metadata generate a new versionId - { uploadId: uploadIdToSimulate, versionId: undefined }); + { uploadId: uploadIdToSimulate, versionId: undefined }, + ); // Store this modified metadata as orphaned object - const putOptions = isVersioned && objMD.versionId - ? { versioning: true } - : {}; + const putOptions = isVersioned && objMD.versionId ? { versioning: true } : {}; await putObjectMDAsync(bucketName, objectKey, orphanedObjectMD, putOptions, log); // Clean up temporary object @@ -600,11 +699,17 @@ describe('Abort MPU - Orphan Cleanup', function testSuite() { bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; - async.series([ - next => s3.send(new CreateBucketCommand({ Bucket: bucketName })).then(() => - next()).catch(err => next(err)), - next => initMetadata(next), - ], done); + async.series( + [ + next => + s3 + .send(new CreateBucketCommand({ Bucket: bucketName })) + .then(() => next()) + .catch(err => next(err)), + next => initMetadata(next), + ], + done, + ); }); afterEach(async () => { @@ -615,19 +720,23 @@ describe('Abort MPU - Orphan Cleanup', function testSuite() { const data = Buffer.from('test data for orphan cleanup'); // Create MPU and upload a part - const createResult = await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucketName, - Key: objectKey, - })); + const createResult = await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucketName, + Key: objectKey, + }), + ); const uploadId = createResult.UploadId; - await s3.send(new UploadPartCommand({ - Bucket: bucketName, - Key: objectKey, - PartNumber: 1, - UploadId: uploadId, - Body: data, - })); + await s3.send( + new UploadPartCommand({ + Bucket: bucketName, + Key: objectKey, + PartNumber: 1, + UploadId: uploadId, + Body: data, + }), + ); // Create realistic orphaned object metadata like a CompleteMPU would when failing before cleanup await createOrphanedObjectMetadata(s3, bucketName, objectKey, uploadId, data, false); @@ -636,11 +745,13 @@ describe('Abort MPU - Orphan Cleanup', function testSuite() { await s3.send(new HeadObjectCommand({ Bucket: bucketName, Key: objectKey })); // Abort MPU - should clean up the orphaned object - await s3.send(new AbortMultipartUploadCommand({ - Bucket: bucketName, - Key: objectKey, - UploadId: uploadId, - })); + await s3.send( + new AbortMultipartUploadCommand({ + Bucket: bucketName, + Key: objectKey, + UploadId: uploadId, + }), + ); // Verify the orphaned object was cleaned up try { @@ -657,63 +768,85 @@ describe('Abort MPU - Orphan Cleanup', function testSuite() { const data = Buffer.from('test versioned orphan cleanup'); // Enable versioning - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: { Status: 'Enabled' }, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ); // Create MPU - const createResult = await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucketName, - Key: objectKey, - })); + const createResult = await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucketName, + Key: objectKey, + }), + ); const uploadId = createResult.UploadId; - await s3.send(new UploadPartCommand({ - Bucket: bucketName, - Key: objectKey, - PartNumber: 1, - UploadId: uploadId, - Body: data, - })); + await s3.send( + new UploadPartCommand({ + Bucket: bucketName, + Key: objectKey, + PartNumber: 1, + UploadId: uploadId, + Body: data, + }), + ); // Create realistic orphaned object metadata like a CompleteMPU would when failing before cleanup const orphanedObjectMD = await createOrphanedObjectMetadata( - s3, bucketName, objectKey, uploadId, data, true); + s3, + bucketName, + objectKey, + uploadId, + data, + true, + ); // Put a new master version on top of the orphaned version // The abort will fetch this during standardMetadataValidateBucketAndObj // It will force abort to findObjectVersionByUploadId - await s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectKey, - Body: 'version 2 data', - })); + await s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectKey, + Body: 'version 2 data', + }), + ); // Verify we have 2 versions (1 regular + 1 orphaned) let listResult = await s3.send(new ListObjectVersionsCommand({ Bucket: bucketName })); let objectVersions = listResult.Versions.filter(v => v.Key === objectKey); - assert.strictEqual(objectVersions.length, 2, - 'Expected 2 versions before abort, 1 regular + 1 orphaned' - ); + assert.strictEqual(objectVersions.length, 2, 'Expected 2 versions before abort, 1 regular + 1 orphaned'); // Abort MPU - should find and clean up only the orphaned version - await s3.send(new AbortMultipartUploadCommand({ - Bucket: bucketName, - Key: objectKey, - UploadId: uploadId, - })); + await s3.send( + new AbortMultipartUploadCommand({ + Bucket: bucketName, + Key: objectKey, + UploadId: uploadId, + }), + ); // Verify only the orphaned version was deleted listResult = await s3.send(new ListObjectVersionsCommand({ Bucket: bucketName })); objectVersions = listResult.Versions.filter(v => v.Key === objectKey); - assert.strictEqual(objectVersions.length, 1, - 'Should have 1 version after abort (orphaned version cleaned up)'); + assert.strictEqual( + objectVersions.length, + 1, + 'Should have 1 version after abort (orphaned version cleaned up)', + ); // ensure orphanedObj doesn't exist try { - await s3.send(new HeadObjectCommand({ Bucket: bucketName, Key: objectKey, - VersionId: orphanedObjectMD.versionId })); + await s3.send( + new HeadObjectCommand({ + Bucket: bucketName, + Key: objectKey, + VersionId: orphanedObjectMD.versionId, + }), + ); assert.fail('Orphaned object should be deleted after abort'); } catch (err) { assert(err); @@ -746,38 +879,48 @@ describe('Abort MPU - Race Conditions', function testSuite() { const data = Buffer.from('test concurrent complete and abort'); // Create MPU and upload part - const createResult = await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucketName, - Key: objectKey, - })); + const createResult = await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucketName, + Key: objectKey, + }), + ); const uploadId = createResult.UploadId; - const uploadResult = await s3.send(new UploadPartCommand({ - Bucket: bucketName, - Key: objectKey, - PartNumber: 1, - UploadId: uploadId, - Body: data, - })); + const uploadResult = await s3.send( + new UploadPartCommand({ + Bucket: bucketName, + Key: objectKey, + PartNumber: 1, + UploadId: uploadId, + Body: data, + }), + ); const etag = uploadResult.ETag; // Start concurrent operations: CompleteMPU and AbortMPU const [completeResult, abortResult] = await Promise.allSettled([ - s3.send(new CompleteMultipartUploadCommand({ - Bucket: bucketName, - Key: objectKey, - UploadId: uploadId, - MultipartUpload: { - Parts: [{ ETag: etag, PartNumber: 1 }], - }, - })), + s3.send( + new CompleteMultipartUploadCommand({ + Bucket: bucketName, + Key: objectKey, + UploadId: uploadId, + MultipartUpload: { + Parts: [{ ETag: etag, PartNumber: 1 }], + }, + }), + ), // Add small delay to create race condition - scheduler.wait(10).then(() => s3.send(new AbortMultipartUploadCommand({ - Bucket: bucketName, - Key: objectKey, - UploadId: uploadId, - }))) + scheduler.wait(10).then(() => + s3.send( + new AbortMultipartUploadCommand({ + Bucket: bucketName, + Key: objectKey, + UploadId: uploadId, + }), + ), + ), ]); // Verify final state is consistent @@ -821,43 +964,51 @@ describe('Abort MPU - Race Conditions', function testSuite() { const data = Buffer.from('test multiple concurrent aborts'); // Create MPU and upload part - const createResult = await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucketName, - Key: objectKey, - })); - const uploadId = createResult.UploadId; - - await s3.send(new UploadPartCommand({ - Bucket: bucketName, - Key: objectKey, - PartNumber: 1, - UploadId: uploadId, - Body: data, - })); - - // Launch 3 concurrent abort operations - const abortResults = await Promise.allSettled([ - s3.send(new AbortMultipartUploadCommand({ - Bucket: bucketName, - Key: objectKey, - UploadId: uploadId, - })), - s3.send(new AbortMultipartUploadCommand({ + const createResult = await s3.send( + new CreateMultipartUploadCommand({ Bucket: bucketName, Key: objectKey, - UploadId: uploadId, - })), - s3.send(new AbortMultipartUploadCommand({ + }), + ); + const uploadId = createResult.UploadId; + + await s3.send( + new UploadPartCommand({ Bucket: bucketName, Key: objectKey, + PartNumber: 1, UploadId: uploadId, - })) + Body: data, + }), + ); + + // Launch 3 concurrent abort operations + const abortResults = await Promise.allSettled([ + s3.send( + new AbortMultipartUploadCommand({ + Bucket: bucketName, + Key: objectKey, + UploadId: uploadId, + }), + ), + s3.send( + new AbortMultipartUploadCommand({ + Bucket: bucketName, + Key: objectKey, + UploadId: uploadId, + }), + ), + s3.send( + new AbortMultipartUploadCommand({ + Bucket: bucketName, + Key: objectKey, + UploadId: uploadId, + }), + ), ]); // Verify results - const abortErrors = abortResults.map(result => - result.status === 'rejected' ? result.reason : null - ); + const abortErrors = abortResults.map(result => (result.status === 'rejected' ? result.reason : null)); // At least one abort should succeed const successfulAborts = abortErrors.filter(err => !err); @@ -883,8 +1034,7 @@ describe('Abort MPU - Race Conditions', function testSuite() { // Verify no MPU metadata remains const listResult = await s3.send(new ListMultipartUploadsCommand({ Bucket: bucketName })); const remainingUploads = (listResult.Uploads || []).filter(upload => upload.UploadId === uploadId); - assert.strictEqual(remainingUploads.length, 0, - 'No MPU metadata should remain after concurrent aborts'); + assert.strictEqual(remainingUploads.length, 0, 'No MPU metadata should remain after concurrent aborts'); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/object/bigMpu.js b/tests/functional/aws-node-sdk/test/object/bigMpu.js index 0570531ea6..f4666fa936 100644 --- a/tests/functional/aws-node-sdk/test/object/bigMpu.js +++ b/tests/functional/aws-node-sdk/test/object/bigMpu.js @@ -2,15 +2,15 @@ const assert = require('assert'); const { timesLimit, waterfall } = require('async'); const { NodeHttpHandler } = require('@smithy/node-http-handler'); -const { +const { S3Client, CreateBucketCommand, - CreateMultipartUploadCommand, - UploadPartCommand, - CompleteMultipartUploadCommand, + CreateMultipartUploadCommand, + UploadPartCommand, + CompleteMultipartUploadCommand, GetObjectCommand, DeleteObjectCommand, - DeleteBucketCommand + DeleteBucketCommand, } = require('@aws-sdk/client-s3'); const getConfig = require('../support/config'); @@ -20,9 +20,10 @@ const key = 'mpuKey'; const body = 'abc'; const partCount = 10000; const eTag = require('crypto').createHash('md5').update(body).digest('hex'); -const finalETag = require('crypto').createHash('md5') - .update(Buffer.from(eTag.repeat(partCount), 'hex').toString('binary'), - 'binary').digest('hex'); +const finalETag = require('crypto') + .createHash('md5') + .update(Buffer.from(eTag.repeat(partCount), 'hex').toString('binary'), 'binary') + .digest('hex'); const partETags = new Array(partCount); function uploadPart(n, uploadId, s3, next) { @@ -36,7 +37,7 @@ function uploadPart(n, uploadId, s3, next) { if (params.PartNumber % 20 === 0) { process.stdout.write(`uploading PartNumber: ${params.PartNumber}\n`); } - + s3.send(new UploadPartCommand(params)) .then(data => { partETags[n] = data.ETag; @@ -65,13 +66,13 @@ describe('large mpu', function tester() { requestTimeout: 0, connectionTimeout: 0, }); - + s3 = new S3Client({ ...config, maxAttempts: 1, requestHandler, }); - + s3.send(new CreateBucketCommand({ Bucket: bucket })) .then(() => done()) .catch(err => done(err)); @@ -90,58 +91,66 @@ describe('large mpu', function tester() { const itSkipIfAWS = process.env.AWS_ON_AIR ? it.skip : it; // will fail on AWS because parts too small - itSkipIfAWS('should intiate, put parts and complete mpu ' + - `with ${partCount} parts`, done => { + itSkipIfAWS('should intiate, put parts and complete mpu ' + `with ${partCount} parts`, done => { process.stdout.write('***Running large MPU test***\n'); - let uploadId; - return waterfall([ - next => { - s3.send(new CreateMultipartUploadCommand({ Bucket: bucket, Key: key })) - .then(data => { - uploadId = data.UploadId; - return next(); - }) - .catch(err => next(err)); - }, - next => timesLimit(partCount, 20, (n, cb) => uploadPart(n, uploadId, s3, cb), err => { - if (err) { - process.stdout.write(`Error in timesLimit: ${err}\n`); - } - return next(err); - }), - next => { - const parts = []; - for (let i = 0; i < partCount; i++) { - if (!partETags[i]) { - return next(new Error(`Missing ETag for part ${i + 1}`)); + let uploadId; + return waterfall( + [ + next => { + s3.send(new CreateMultipartUploadCommand({ Bucket: bucket, Key: key })) + .then(data => { + uploadId = data.UploadId; + return next(); + }) + .catch(err => next(err)); + }, + next => + timesLimit( + partCount, + 20, + (n, cb) => uploadPart(n, uploadId, s3, cb), + err => { + if (err) { + process.stdout.write(`Error in timesLimit: ${err}\n`); + } + return next(err); + }, + ), + next => { + const parts = []; + for (let i = 0; i < partCount; i++) { + if (!partETags[i]) { + return next(new Error(`Missing ETag for part ${i + 1}`)); + } + parts.push({ + ETag: partETags[i], + PartNumber: i + 1, + }); } - parts.push({ - ETag: partETags[i], - PartNumber: i + 1, - }); - } - const params = { - Bucket: bucket, - Key: key, - UploadId: uploadId, - MultipartUpload: { - Parts: parts, - }, - }; - return s3.send(new CompleteMultipartUploadCommand(params)) - .then(() => next()) - .catch(err => next(err)); - }, - next => { - s3.send(new GetObjectCommand({ Bucket: bucket, Key: key })) - .then(data => { - assert.strictEqual(data.ETag, - `"${finalETag}-${partCount}"`); - process.stdout.write('get object successful\n'); - return next(); - }) - .catch(err => next(err)); - }, - ], done); + const params = { + Bucket: bucket, + Key: key, + UploadId: uploadId, + MultipartUpload: { + Parts: parts, + }, + }; + return s3 + .send(new CompleteMultipartUploadCommand(params)) + .then(() => next()) + .catch(err => next(err)); + }, + next => { + s3.send(new GetObjectCommand({ Bucket: bucket, Key: key })) + .then(data => { + assert.strictEqual(data.ETag, `"${finalETag}-${partCount}"`); + process.stdout.write('get object successful\n'); + return next(); + }) + .catch(err => next(err)); + }, + ], + done, + ); }); }); diff --git a/tests/functional/aws-node-sdk/test/object/completeMPU.js b/tests/functional/aws-node-sdk/test/object/completeMPU.js index f51a1d64c3..b78ca397b5 100644 --- a/tests/functional/aws-node-sdk/test/object/completeMPU.js +++ b/tests/functional/aws-node-sdk/test/object/completeMPU.js @@ -2,11 +2,7 @@ const assert = require('assert'); const async = require('async'); const withV4 = require('../support/withV4'); const BucketUtility = require('../../lib/utility/bucket-util'); -const { - removeAllVersions, - versioningEnabled, - versioningSuspended, -} = require('../../lib/utility/versioning-util.js'); +const { removeAllVersions, versioningEnabled, versioningSuspended } = require('../../lib/utility/versioning-util.js'); const { taggingTests } = require('../../lib/utility/tagging'); const { CreateBucketCommand, @@ -17,14 +13,13 @@ const { GetObjectCommand, PutBucketVersioningCommand, GetObjectTaggingCommand, - NoSuchKey + NoSuchKey, } = require('@aws-sdk/client-s3'); const date = Date.now(); const bucket = `completempu${date}`; const key = 'key'; - describe('Complete MPU', () => { withV4(sigCfg => { const bucketUtil = new BucketUtility('default', sigCfg); @@ -32,59 +27,68 @@ describe('Complete MPU', () => { function _completeMpuAndCheckVid(uploadId, eTag, expectedVid, cb) { let versionId; - s3.send(new CompleteMultipartUploadCommand({ - Bucket: bucket, - Key: key, - MultipartUpload: { - Parts: [{ ETag: eTag, PartNumber: 1 }], - }, - UploadId: uploadId - })) - .then(data => { - versionId = data.VersionId; - if (expectedVid) { - assert.notEqual(versionId, undefined); - } else { - assert.strictEqual(versionId, expectedVid); - } - return s3.send(new GetObjectCommand({ + s3.send( + new CompleteMultipartUploadCommand({ Bucket: bucket, Key: key, - })); - }) - .then(data => { - if (versionId) { - assert.strictEqual(data.VersionId, versionId); - } - cb(); - }) - .catch(cb); + MultipartUpload: { + Parts: [{ ETag: eTag, PartNumber: 1 }], + }, + UploadId: uploadId, + }), + ) + .then(data => { + versionId = data.VersionId; + if (expectedVid) { + assert.notEqual(versionId, undefined); + } else { + assert.strictEqual(versionId, expectedVid); + } + return s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + }), + ); + }) + .then(data => { + if (versionId) { + assert.strictEqual(data.VersionId, versionId); + } + cb(); + }) + .catch(cb); } function _initiateMpuAndPutOnePart() { const result = {}; - return s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: key - })) - .then(data => { - result.uploadId = data.UploadId; - return s3.send(new UploadPartCommand({ - Bucket: bucket, - Key: key, - PartNumber: 1, - UploadId: data.UploadId, - Body: 'foo', - })); - }) - .then(data => { - result.eTag = data.ETag; - return result; - }) - .catch(err => { - process.stdout.write(`Error in beforeEach: ${err}\n`); - throw err; - }); + return s3 + .send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + }), + ) + .then(data => { + result.uploadId = data.UploadId; + return s3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: key, + PartNumber: 1, + UploadId: data.UploadId, + Body: 'foo', + }), + ); + }) + .then(data => { + result.eTag = data.ETag; + return result; + }) + .catch(err => { + process.stdout.write(`Error in beforeEach: ${err}\n`); + throw err; + }); } beforeEach(async () => { @@ -109,59 +113,74 @@ describe('Complete MPU', () => { let uploadId; let eTag; - beforeEach(() => _initiateMpuAndPutOnePart() - .then(result => { + beforeEach(() => + _initiateMpuAndPutOnePart().then(result => { uploadId = result.uploadId; eTag = result.eTag; - }) + }), ); - it('should complete an MPU with fewer parts than were ' + - 'originally put without returning a version id', done => { - _completeMpuAndCheckVid(uploadId, eTag, undefined, done); - }); + it( + 'should complete an MPU with fewer parts than were ' + 'originally put without returning a version id', + done => { + _completeMpuAndCheckVid(uploadId, eTag, undefined, done); + }, + ); }); describe('on bucket with enabled versioning', () => { let uploadId; let eTag; - beforeEach(() => s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningEnabled - })) - .then(() => _initiateMpuAndPutOnePart()) - .then(result => { - uploadId = result.uploadId; - eTag = result.eTag; - }) + beforeEach(() => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningEnabled, + }), + ) + .then(() => _initiateMpuAndPutOnePart()) + .then(result => { + uploadId = result.uploadId; + eTag = result.eTag; + }), ); - it('should complete an MPU with fewer parts than were ' + - 'originally put and return a version id', done => { - _completeMpuAndCheckVid(uploadId, eTag, true, done); - }); + it( + 'should complete an MPU with fewer parts than were ' + 'originally put and return a version id', + done => { + _completeMpuAndCheckVid(uploadId, eTag, true, done); + }, + ); }); describe('on bucket with suspended versioning', () => { let uploadId; let eTag; - beforeEach(() => s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningSuspended - })) - .then(() => _initiateMpuAndPutOnePart()) - .then(result => { - uploadId = result.uploadId; - eTag = result.eTag; - }) + beforeEach(() => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningSuspended, + }), + ) + .then(() => _initiateMpuAndPutOnePart()) + .then(result => { + uploadId = result.uploadId; + eTag = result.eTag; + }), ); - it('should complete an MPU with fewer parts than were ' + - 'originally put and should not return a version id', done => { - _completeMpuAndCheckVid(uploadId, eTag, undefined, done); - }); + it( + 'should complete an MPU with fewer parts than were ' + + 'originally put and should not return a version id', + done => { + _completeMpuAndCheckVid(uploadId, eTag, undefined, done); + }, + ); }); describe('with tags set on initiation', () => { @@ -169,78 +188,91 @@ describe('Complete MPU', () => { taggingTests.forEach(test => { it(test.it, done => { - const [key, value] = - [test.tag.key, test.tag.value].map(encodeURIComponent); + const [key, value] = [test.tag.key, test.tag.value].map(encodeURIComponent); const tagging = `${key}=${value}`; - async.waterfall([ - next => { - s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: tagKey, - Tagging: tagging, - })) - .then(data => { - if (test.error) { - return next(new Error('Expected error but got success')); - } - return next(null, data.UploadId); - }) - .catch(err => { - if (test.error) { - assert.strictEqual(err.name, test.error); - assert.strictEqual(err.$metadata.httpStatusCode, 400); - return next('expected'); - } - return next(err); - }); - }, - (uploadId, next) => { - s3.send(new UploadPartCommand({ - Bucket: bucket, - Key: tagKey, - PartNumber: 1, - UploadId: uploadId, - Body: 'foo', - })) - .then(data => next(null, data.ETag, uploadId)) - .catch(err => next(err)); - }, - (eTag, uploadId, next) => { - s3.send(new CompleteMultipartUploadCommand({ - Bucket: bucket, - Key: tagKey, - UploadId: uploadId, - MultipartUpload: { - Parts: [{ - ETag: eTag, + async.waterfall( + [ + next => { + s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: tagKey, + Tagging: tagging, + }), + ) + .then(data => { + if (test.error) { + return next(new Error('Expected error but got success')); + } + return next(null, data.UploadId); + }) + .catch(err => { + if (test.error) { + assert.strictEqual(err.name, test.error); + assert.strictEqual(err.$metadata.httpStatusCode, 400); + return next('expected'); + } + return next(err); + }); + }, + (uploadId, next) => { + s3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: tagKey, PartNumber: 1, - }], - }, - })) - .then(() => next()) - .catch(err => next(err)); - }, - ], err => { - if (err === 'expected') { - done(); - } else { - assert.ifError(err); - s3.send(new GetObjectTaggingCommand({ - Bucket: bucket, - Key: tagKey, - })) - .then(tagData => { - assert.deepStrictEqual(tagData.TagSet, - [{ - Key: test.tag.key, - Value: test.tag.value, - }]); + UploadId: uploadId, + Body: 'foo', + }), + ) + .then(data => next(null, data.ETag, uploadId)) + .catch(err => next(err)); + }, + (eTag, uploadId, next) => { + s3.send( + new CompleteMultipartUploadCommand({ + Bucket: bucket, + Key: tagKey, + UploadId: uploadId, + MultipartUpload: { + Parts: [ + { + ETag: eTag, + PartNumber: 1, + }, + ], + }, + }), + ) + .then(() => next()) + .catch(err => next(err)); + }, + ], + err => { + if (err === 'expected') { done(); - }) - .catch(err => done(err)); - } - }); + } else { + assert.ifError(err); + s3.send( + new GetObjectTaggingCommand({ + Bucket: bucket, + Key: tagKey, + }), + ) + .then(tagData => { + assert.deepStrictEqual(tagData.TagSet, [ + { + Key: test.tag.key, + Value: test.tag.value, + }, + ]); + done(); + }) + .catch(err => done(err)); + } + }, + ); }); }); }); @@ -249,37 +281,41 @@ describe('Complete MPU', () => { let uploadId; let eTag; - beforeEach(() => _initiateMpuAndPutOnePart() - .then(result => { + beforeEach(() => + _initiateMpuAndPutOnePart().then(result => { uploadId = result.uploadId; eTag = result.eTag; - }) + }), ); it('should complete the MPU successfully and leave a readable object', done => { - async.parallel([ - doneReUpload => { - s3.send(new UploadPartCommand({ - Bucket: bucket, - Key: key, - PartNumber: 1, - UploadId: uploadId, - Body: 'foo', - })) - .then(() => doneReUpload()) - .catch(err => { - // in case the CompleteMPU finished earlier, - // we may get a NoSuchKey error, so just - // ignore it - if (err instanceof NoSuchKey) { - return doneReUpload(); - } - return doneReUpload(err); - }); - }, - doneComplete => _completeMpuAndCheckVid( - uploadId, eTag, undefined, doneComplete), - ], done); + async.parallel( + [ + doneReUpload => { + s3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: key, + PartNumber: 1, + UploadId: uploadId, + Body: 'foo', + }), + ) + .then(() => doneReUpload()) + .catch(err => { + // in case the CompleteMPU finished earlier, + // we may get a NoSuchKey error, so just + // ignore it + if (err instanceof NoSuchKey) { + return doneReUpload(); + } + return doneReUpload(err); + }); + }, + doneComplete => _completeMpuAndCheckVid(uploadId, eTag, undefined, doneComplete), + ], + done, + ); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/object/compluteMpu.js b/tests/functional/aws-node-sdk/test/object/compluteMpu.js index d8a20a6c7a..339b2407e0 100644 --- a/tests/functional/aws-node-sdk/test/object/compluteMpu.js +++ b/tests/functional/aws-node-sdk/test/object/compluteMpu.js @@ -47,13 +47,14 @@ describe('aws-node-sdk test bucket complete mpu', () => { Parts: parts, }, }; - s3.send(new CompleteMultipartUploadCommand(params)).then(() => { - done('accepted xml body larger than 1 MB'); - }).catch(error => { - assert.strictEqual(error.$metadata.httpStatusCode, 400); - assert.strictEqual( - error.name, 'InvalidRequest'); - done(); - }); + s3.send(new CompleteMultipartUploadCommand(params)) + .then(() => { + done('accepted xml body larger than 1 MB'); + }) + .catch(error => { + assert.strictEqual(error.$metadata.httpStatusCode, 400); + assert.strictEqual(error.name, 'InvalidRequest'); + done(); + }); }); }); diff --git a/tests/functional/aws-node-sdk/test/object/copyPart.js b/tests/functional/aws-node-sdk/test/object/copyPart.js index 52aec224d9..302e4c9f63 100644 --- a/tests/functional/aws-node-sdk/test/object/copyPart.js +++ b/tests/functional/aws-node-sdk/test/object/copyPart.js @@ -1,7 +1,8 @@ const assert = require('assert'); const crypto = require('crypto'); -const { CreateBucketCommand, +const { + CreateBucketCommand, PutObjectCommand, GetObjectCommand, HeadObjectCommand, @@ -10,13 +11,12 @@ const { CreateBucketCommand, UploadPartCopyCommand, CompleteMultipartUploadCommand, AbortMultipartUploadCommand, - PutObjectAclCommand + PutObjectAclCommand, } = require('@aws-sdk/client-s3'); const withV4 = require('../support/withV4'); const BucketUtility = require('../../lib/utility/bucket-util'); -const { createEncryptedBucketPromise } = - require('../../lib/utility/createEncryptedBucket'); +const { createEncryptedBucketPromise } = require('../../lib/utility/createEncryptedBucket'); const { fakeMetadataTransition, fakeMetadataArchive } = require('../utils/init'); const { hasColdStorage } = require('../../lib/utility/test-utils'); @@ -33,8 +33,7 @@ const otherAccountS3 = otherAccountBucketUtility.s3; const oneHundredMBPlus11 = 110100481; function checkNoError(err) { - assert.equal(err, null, - `Expected success, got error ${JSON.stringify(err)}`); + assert.equal(err, null, `Expected success, got error ${JSON.stringify(err)}`); } function checkError(err, code) { @@ -56,240 +55,350 @@ describe('Object Part Copy', () => { if (process.env.ENABLE_KMS_ENCRYPTION === 'true') { s3.createBucketPromise = createEncryptedBucketPromise; } - return s3.createBucketPromise({ Bucket: sourceBucketName }) - .catch(err => { - process.stdout.write(`Error creating source bucket: ${err}\n`); - throw err; - }).then(() => - s3.createBucketPromise({ Bucket: destBucketName }) - ).catch(err => { - process.stdout.write(`Error creating dest bucket: ${err}\n`); - throw err; - }) - .then(() => s3.send(new PutObjectCommand({ - Bucket: sourceBucketName, - Key: sourceObjName, - Body: content, - }))) - .then(res => { - etag = res.ETag; - return s3.send(new HeadObjectCommand({ - Bucket: sourceBucketName, - Key: sourceObjName, - })); - }).then(() => s3.send(new CreateMultipartUploadCommand({ - Bucket: destBucketName, - Key: destObjName, - })).then(initiateRes => { - uploadId = initiateRes.UploadId; - })).catch(err => { - process.stdout.write(`Error in outer beforeEach: ${err}\n`); - throw err; - }); - }); - - afterEach(() => bucketUtil.empty(sourceBucketName) - .then(() => bucketUtil.empty(destBucketName)) - .then(() => s3.send(new AbortMultipartUploadCommand({ - Bucket: destBucketName, - Key: destObjName, - UploadId: uploadId, - }))).catch(err => { - if (err.name !== 'NoSuchUpload') { - process.stdout.write(`Error in afterEach: ${err}\n`); + return s3 + .createBucketPromise({ Bucket: sourceBucketName }) + .catch(err => { + process.stdout.write(`Error creating source bucket: ${err}\n`); throw err; - } - }) - .then(() => bucketUtil.deleteMany([sourceBucketName, - destBucketName])) - ); + }) + .then(() => s3.createBucketPromise({ Bucket: destBucketName })) + .catch(err => { + process.stdout.write(`Error creating dest bucket: ${err}\n`); + throw err; + }) + .then(() => + s3.send( + new PutObjectCommand({ + Bucket: sourceBucketName, + Key: sourceObjName, + Body: content, + }), + ), + ) + .then(res => { + etag = res.ETag; + return s3.send( + new HeadObjectCommand({ + Bucket: sourceBucketName, + Key: sourceObjName, + }), + ); + }) + .then(() => + s3 + .send( + new CreateMultipartUploadCommand({ + Bucket: destBucketName, + Key: destObjName, + }), + ) + .then(initiateRes => { + uploadId = initiateRes.UploadId; + }), + ) + .catch(err => { + process.stdout.write(`Error in outer beforeEach: ${err}\n`); + throw err; + }); + }); + afterEach(() => + bucketUtil + .empty(sourceBucketName) + .then(() => bucketUtil.empty(destBucketName)) + .then(() => + s3.send( + new AbortMultipartUploadCommand({ + Bucket: destBucketName, + Key: destObjName, + UploadId: uploadId, + }), + ), + ) + .catch(err => { + if (err.name !== 'NoSuchUpload') { + process.stdout.write(`Error in afterEach: ${err}\n`); + throw err; + } + }) + .then(() => bucketUtil.deleteMany([sourceBucketName, destBucketName])), + ); - it('should copy a part from a source bucket to a different ' + - 'destination bucket', () => s3.send(new UploadPartCopyCommand({ Bucket: destBucketName, - Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - PartNumber: 1, - UploadId: uploadId, - })).then(res => { - assert.strictEqual(res.CopyPartResult.ETag, etag); - assert(res.CopyPartResult.LastModified); - })); - - it('should copy a part from a source bucket to a different ' + - 'destination bucket and complete the MPU', () => s3.send(new UploadPartCopyCommand({ Bucket: destBucketName, - Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - PartNumber: 1, - UploadId: uploadId, - })).then(res => { - assert.strictEqual(res.CopyPartResult.ETag, etag); - assert(res.CopyPartResult.LastModified); - return s3.send(new CompleteMultipartUploadCommand({ - Bucket: destBucketName, - Key: destObjName, + it('should copy a part from a source bucket to a different ' + 'destination bucket', () => + s3 + .send( + new UploadPartCopyCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + PartNumber: 1, UploadId: uploadId, - MultipartUpload: { - Parts: [ - { ETag: etag, PartNumber: 1 }, - ], - }, - })).then(res => { - assert.strictEqual(res.Bucket, destBucketName); - assert.strictEqual(res.Key, destObjName); - // AWS confirmed final ETag for MPU - assert.strictEqual(res.ETag, - '"db77ebbae9e9f5a244a26b86193ad818-1"'); - }); - })); - - it('should return InvalidArgument error given invalid range', () => s3.send(new PutObjectCommand({ - Bucket: sourceBucketName, - Key: sourceObjName, - Body: Buffer.alloc(oneHundredMBPlus11, 'packing'), - })).then(() => s3.send(new UploadPartCopyCommand({ - Bucket: destBucketName, - Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - PartNumber: 1, - UploadId: uploadId, - CopySourceRange: 'bad-range-parameter', - })).catch(err => { - checkError(err, 'InvalidArgument'); - }))); - - it('should return EntityTooLarge error if attempt to copy ' + - 'object larger than max and do not specify smaller ' + - 'range in request', () => s3.send(new PutObjectCommand({ - Bucket: sourceBucketName, - Key: sourceObjName, - Body: Buffer.alloc(oneHundredMBPlus11, 'packing'), - })).then(() => s3.send(new UploadPartCopyCommand({ - Bucket: destBucketName, - Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - PartNumber: 1, - UploadId: uploadId, - }))).catch(err => { - checkError(err, 'EntityTooLarge'); - })); - - it('should return EntityTooLarge error if attempt to copy ' + - 'object larger than max and specify too large ' + - 'range in request', () => s3.send(new PutObjectCommand({ - Bucket: sourceBucketName, - Key: sourceObjName, - Body: Buffer.alloc(oneHundredMBPlus11, 'packing'), - })).then(() => s3.send(new UploadPartCopyCommand({ - Bucket: destBucketName, - Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - PartNumber: 1, - UploadId: uploadId, - CopySourceRange: `bytes=0-${oneHundredMBPlus11}`, - }))).catch(err => { - checkError(err, 'EntityTooLarge'); - })); - - it('should succeed if attempt to copy ' + - 'object larger than max but specify acceptable ' + - 'range in request', () => s3.send(new PutObjectCommand({ - Bucket: sourceBucketName, - Key: sourceObjName, - Body: Buffer.alloc(oneHundredMBPlus11, 'packing'), - })).then(() => s3.send(new UploadPartCopyCommand({ - Bucket: destBucketName, - Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - PartNumber: 1, - UploadId: uploadId, - CopySourceRange: 'bytes=0-100', - }))).catch(err => { - checkNoError(err); - })); - - it('should copy a 0 byte object part from a source bucket to a ' + - 'different destination bucket and complete the MPU', () => { - const emptyFileETag = '"d41d8cd98f00b204e9800998ecf8427e"'; - return s3.send(new PutObjectCommand({ - Bucket: sourceBucketName, - Key: sourceObjName, - Body: '', - })).then(() => s3.send(new UploadPartCopyCommand({ - Bucket: destBucketName, - Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - PartNumber: 1, - UploadId: uploadId, - })).then(res => { - assert.strictEqual(res.CopyPartResult.ETag, emptyFileETag); + }), + ) + .then(res => { + assert.strictEqual(res.CopyPartResult.ETag, etag); assert(res.CopyPartResult.LastModified); - return s3.send(new CompleteMultipartUploadCommand({ + }), + ); + + it('should copy a part from a source bucket to a different ' + 'destination bucket and complete the MPU', () => + s3 + .send( + new UploadPartCopyCommand({ Bucket: destBucketName, Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + PartNumber: 1, UploadId: uploadId, - MultipartUpload: { - Parts: [ - { ETag: emptyFileETag, PartNumber: 1 }, - ], - }, - })).then(res => { - assert.strictEqual(res.Bucket, destBucketName); - assert.strictEqual(res.Key, destObjName); + }), + ) + .then(res => { + assert.strictEqual(res.CopyPartResult.ETag, etag); + assert(res.CopyPartResult.LastModified); + return s3 + .send( + new CompleteMultipartUploadCommand({ + Bucket: destBucketName, + Key: destObjName, + UploadId: uploadId, + MultipartUpload: { + Parts: [{ ETag: etag, PartNumber: 1 }], + }, + }), + ) + .then(res => { + assert.strictEqual(res.Bucket, destBucketName); + assert.strictEqual(res.Key, destObjName); // AWS confirmed final ETag for MPU - assert.strictEqual(res.ETag,'"59adb24ef3cdbe0297f05b395827453f-1"'); - }); - })); - }); + assert.strictEqual(res.ETag, '"db77ebbae9e9f5a244a26b86193ad818-1"'); + }); + }), + ); - it('should copy a part using a range header from a source bucket ' + - 'to a different destination bucket and complete the MPU', () => { - const rangeETag = '"ac1be00f1f162e20d58099eec2ea1c70"'; - // AWS confirmed final ETag for MPU - const finalMpuETag = '"bff2a6af3adfd8e107a06de01d487176-1"'; - return s3.send(new UploadPartCopyCommand({ - Bucket: destBucketName, - Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - PartNumber: 1, - CopySourceRange: 'bytes=0-3', - UploadId: uploadId, - })).then(res => { - assert.strictEqual(res.CopyPartResult.ETag, rangeETag); - assert(res.CopyPartResult.LastModified); - return s3.send(new CompleteMultipartUploadCommand({ - Bucket: destBucketName, - Key: destObjName, - UploadId: uploadId, - MultipartUpload: { - Parts: [ - { ETag: rangeETag, PartNumber: 1 }, - ], - }, - })).then(res => { - assert.strictEqual(res.Bucket, destBucketName); - assert.strictEqual(res.Key, destObjName); - assert.strictEqual(res.ETag, finalMpuETag); - return s3.send(new GetObjectCommand({ - Bucket: destBucketName, - Key: destObjName, - })).then(async res => { - assert.strictEqual(res.ETag, finalMpuETag); - assert.strictEqual(res.ContentLength, 4); - const body = await res.Body.transformToString(); - assert.strictEqual(body, 'I am'); + it('should return InvalidArgument error given invalid range', () => + s3 + .send( + new PutObjectCommand({ + Bucket: sourceBucketName, + Key: sourceObjName, + Body: Buffer.alloc(oneHundredMBPlus11, 'packing'), + }), + ) + .then(() => + s3 + .send( + new UploadPartCopyCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + PartNumber: 1, + UploadId: uploadId, + CopySourceRange: 'bad-range-parameter', + }), + ) + .catch(err => { + checkError(err, 'InvalidArgument'); + }), + )); + + it( + 'should return EntityTooLarge error if attempt to copy ' + + 'object larger than max and do not specify smaller ' + + 'range in request', + () => + s3 + .send( + new PutObjectCommand({ + Bucket: sourceBucketName, + Key: sourceObjName, + Body: Buffer.alloc(oneHundredMBPlus11, 'packing'), + }), + ) + .then(() => + s3.send( + new UploadPartCopyCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + PartNumber: 1, + UploadId: uploadId, + }), + ), + ) + .catch(err => { + checkError(err, 'EntityTooLarge'); + }), + ); + + it( + 'should return EntityTooLarge error if attempt to copy ' + + 'object larger than max and specify too large ' + + 'range in request', + () => + s3 + .send( + new PutObjectCommand({ + Bucket: sourceBucketName, + Key: sourceObjName, + Body: Buffer.alloc(oneHundredMBPlus11, 'packing'), + }), + ) + .then(() => + s3.send( + new UploadPartCopyCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + PartNumber: 1, + UploadId: uploadId, + CopySourceRange: `bytes=0-${oneHundredMBPlus11}`, + }), + ), + ) + .catch(err => { + checkError(err, 'EntityTooLarge'); + }), + ); + + it( + 'should succeed if attempt to copy ' + + 'object larger than max but specify acceptable ' + + 'range in request', + () => + s3 + .send( + new PutObjectCommand({ + Bucket: sourceBucketName, + Key: sourceObjName, + Body: Buffer.alloc(oneHundredMBPlus11, 'packing'), + }), + ) + .then(() => + s3.send( + new UploadPartCopyCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + PartNumber: 1, + UploadId: uploadId, + CopySourceRange: 'bytes=0-100', + }), + ), + ) + .catch(err => { + checkNoError(err); + }), + ); + + it( + 'should copy a 0 byte object part from a source bucket to a ' + + 'different destination bucket and complete the MPU', + () => { + const emptyFileETag = '"d41d8cd98f00b204e9800998ecf8427e"'; + return s3 + .send( + new PutObjectCommand({ + Bucket: sourceBucketName, + Key: sourceObjName, + Body: '', + }), + ) + .then(() => + s3 + .send( + new UploadPartCopyCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + PartNumber: 1, + UploadId: uploadId, + }), + ) + .then(res => { + assert.strictEqual(res.CopyPartResult.ETag, emptyFileETag); + assert(res.CopyPartResult.LastModified); + return s3 + .send( + new CompleteMultipartUploadCommand({ + Bucket: destBucketName, + Key: destObjName, + UploadId: uploadId, + MultipartUpload: { + Parts: [{ ETag: emptyFileETag, PartNumber: 1 }], + }, + }), + ) + .then(res => { + assert.strictEqual(res.Bucket, destBucketName); + assert.strictEqual(res.Key, destObjName); + // AWS confirmed final ETag for MPU + assert.strictEqual(res.ETag, '"59adb24ef3cdbe0297f05b395827453f-1"'); + }); + }), + ); + }, + ); + + it( + 'should copy a part using a range header from a source bucket ' + + 'to a different destination bucket and complete the MPU', + () => { + const rangeETag = '"ac1be00f1f162e20d58099eec2ea1c70"'; + // AWS confirmed final ETag for MPU + const finalMpuETag = '"bff2a6af3adfd8e107a06de01d487176-1"'; + return s3 + .send( + new UploadPartCopyCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + PartNumber: 1, + CopySourceRange: 'bytes=0-3', + UploadId: uploadId, + }), + ) + .then(res => { + assert.strictEqual(res.CopyPartResult.ETag, rangeETag); + assert(res.CopyPartResult.LastModified); + return s3 + .send( + new CompleteMultipartUploadCommand({ + Bucket: destBucketName, + Key: destObjName, + UploadId: uploadId, + MultipartUpload: { + Parts: [{ ETag: rangeETag, PartNumber: 1 }], + }, + }), + ) + .then(res => { + assert.strictEqual(res.Bucket, destBucketName); + assert.strictEqual(res.Key, destObjName); + assert.strictEqual(res.ETag, finalMpuETag); + return s3 + .send( + new GetObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + }), + ) + .then(async res => { + assert.strictEqual(res.ETag, finalMpuETag); + assert.strictEqual(res.ContentLength, 4); + const body = await res.Body.transformToString(); + assert.strictEqual(body, 'I am'); + }); + }); }); - }); - }); - }); + }, + ); describe('When copy source was put by MPU', () => { let sourceMpuId; const sourceMpuKey = 'sourceMpuKey'; // total hash for sourceMpuKey when MPU completed // (confirmed with AWS) - const totalMpuObjectHash = - '"9b0de95bd76728c778b9e25fd7ce2ef7"'; + const totalMpuObjectHash = '"9b0de95bd76728c778b9e25fd7ce2ef7"'; beforeEach(() => { const parts = []; @@ -301,379 +410,494 @@ describe('Object Part Copy', () => { const otherPartBuff = Buffer.alloc(5242880, 1); otherMd5HashPart.update(otherPartBuff); const otherPartHash = otherMd5HashPart.digest('hex'); - return s3.send(new CreateMultipartUploadCommand({ - Bucket: sourceBucketName, - Key: sourceMpuKey, - })).then(initiateRes => { - sourceMpuId = initiateRes.UploadId; - }).catch(err => { - process.stdout.write(`Error initiating MPU in MPU beforeEach: ${err}\n`); - throw err; - }).then(() => { - const partUploads = []; - // Concurrent uploads help trigger flakiness with "TimeoutError: - // socket hang up" due to a keep-alive race: the server closes - // an idle connection just as the client picks it from the pool. - const uploadWithRetry = (params, attempt = 0) => - s3.send(new UploadPartCommand(params)).catch(err => { - if (attempt < 3) { - process.stdout.write(`Retrying UploadPart ${params.PartNumber} ` - + `(attempt ${attempt + 1}/3): ${err}\n`); - return uploadWithRetry(params, attempt + 1); - } - throw err; - }); - for (let i = 1; i < 10; i++) { - const partBuffHere = i % 2 ? partBuff : otherPartBuff; - const partHashHere = i % 2 ? partHash : otherPartHash; - partUploads.push(uploadWithRetry({ + return s3 + .send( + new CreateMultipartUploadCommand({ Bucket: sourceBucketName, Key: sourceMpuKey, - PartNumber: i, - UploadId: sourceMpuId, - Body: partBuffHere, - })); - parts.push({ - ETag: partHashHere, - PartNumber: i, - }); - } - process.stdout.write('about to put parts\n'); - return Promise.all(partUploads); - }).catch(err => { - process.stdout.write(`Error putting parts in MPU beforeEach: ${err}\n`); - throw err; - }).then(() => { - process.stdout.write('completing mpu\n'); - return s3.send(new CompleteMultipartUploadCommand({ - Bucket: sourceBucketName, - Key: sourceMpuKey, - UploadId: sourceMpuId, - MultipartUpload: { - Parts: parts, - }, - })); - }).then(() => { - process.stdout.write('finished completing mpu\n'); - }).catch(err => { - process.stdout.write(`Error in MPU beforeEach: ${err}\n`); - throw err; - }); + }), + ) + .then(initiateRes => { + sourceMpuId = initiateRes.UploadId; + }) + .catch(err => { + process.stdout.write(`Error initiating MPU in MPU beforeEach: ${err}\n`); + throw err; + }) + .then(() => { + const partUploads = []; + // Concurrent uploads help trigger flakiness with "TimeoutError: + // socket hang up" due to a keep-alive race: the server closes + // an idle connection just as the client picks it from the pool. + const uploadWithRetry = (params, attempt = 0) => + s3.send(new UploadPartCommand(params)).catch(err => { + if (attempt < 3) { + process.stdout.write( + `Retrying UploadPart ${params.PartNumber} ` + + `(attempt ${attempt + 1}/3): ${err}\n`, + ); + return uploadWithRetry(params, attempt + 1); + } + throw err; + }); + for (let i = 1; i < 10; i++) { + const partBuffHere = i % 2 ? partBuff : otherPartBuff; + const partHashHere = i % 2 ? partHash : otherPartHash; + partUploads.push( + uploadWithRetry({ + Bucket: sourceBucketName, + Key: sourceMpuKey, + PartNumber: i, + UploadId: sourceMpuId, + Body: partBuffHere, + }), + ); + parts.push({ + ETag: partHashHere, + PartNumber: i, + }); + } + process.stdout.write('about to put parts\n'); + return Promise.all(partUploads); + }) + .catch(err => { + process.stdout.write(`Error putting parts in MPU beforeEach: ${err}\n`); + throw err; + }) + .then(() => { + process.stdout.write('completing mpu\n'); + return s3.send( + new CompleteMultipartUploadCommand({ + Bucket: sourceBucketName, + Key: sourceMpuKey, + UploadId: sourceMpuId, + MultipartUpload: { + Parts: parts, + }, + }), + ); + }) + .then(() => { + process.stdout.write('finished completing mpu\n'); + }) + .catch(err => { + process.stdout.write(`Error in MPU beforeEach: ${err}\n`); + throw err; + }); }); - afterEach(() => s3.send(new AbortMultipartUploadCommand({ - Bucket: sourceBucketName, - Key: sourceMpuKey, - UploadId: sourceMpuId, - })).catch(err => { - if (err.name !== 'NoSuchUpload' - && err.name !== 'NoSuchBucket') { - process.stdout.write(`Error in afterEach: ${err}\n`); - throw err; - } - })); + afterEach(() => + s3 + .send( + new AbortMultipartUploadCommand({ + Bucket: sourceBucketName, + Key: sourceMpuKey, + UploadId: sourceMpuId, + }), + ) + .catch(err => { + if (err.name !== 'NoSuchUpload' && err.name !== 'NoSuchBucket') { + process.stdout.write(`Error in afterEach: ${err}\n`); + throw err; + } + }), + ); - it('should copy a part from a source bucket to a different ' + - 'destination bucket', () => { + it('should copy a part from a source bucket to a different ' + 'destination bucket', () => { process.stdout.write('Entered first mpu test\n'); - return s3.send(new UploadPartCopyCommand({ - Bucket: destBucketName, - Key: destObjName, - CopySource: `${sourceBucketName}/${sourceMpuKey}`, - PartNumber: 1, - UploadId: uploadId, - })).then(res => { - assert.strictEqual(res.CopyPartResult.ETag, - totalMpuObjectHash); - assert(res.CopyPartResult.LastModified); - }); - }); - - it('should copy two parts from a source bucket to a different ' + - 'destination bucket and complete the MPU', () => { - process.stdout.write('Putting first part in MPU test\n'); - return s3.send(new UploadPartCopyCommand({ - Bucket: destBucketName, - Key: destObjName, - CopySource: `${sourceBucketName}/${sourceMpuKey}`, - PartNumber: 1, - UploadId: uploadId, - })).then(res => { - assert.strictEqual(res.CopyPartResult.ETag, totalMpuObjectHash); - assert(res.CopyPartResult.LastModified); - }).then(() => { - process.stdout.write('Putting second part in MPU test\n'); - return s3.send(new UploadPartCopyCommand({ - Bucket: destBucketName, - Key: destObjName, - CopySource: `${sourceBucketName}/${sourceMpuKey}`, - PartNumber: 2, - UploadId: uploadId, - })).then(res => { - assert.strictEqual(res.CopyPartResult.ETag, totalMpuObjectHash); - assert(res.CopyPartResult.LastModified); - }).then(() => { - process.stdout.write('Completing MPU\n'); - return s3.send(new CompleteMultipartUploadCommand({ + return s3 + .send( + new UploadPartCopyCommand({ Bucket: destBucketName, Key: destObjName, + CopySource: `${sourceBucketName}/${sourceMpuKey}`, + PartNumber: 1, UploadId: uploadId, - MultipartUpload: { - Parts: [ - { ETag: totalMpuObjectHash, PartNumber: 1 }, - { ETag: totalMpuObjectHash, PartNumber: 2 }, - ], - }, - })).then(res => { - assert.strictEqual(res.Bucket, destBucketName); - assert.strictEqual(res.Key, destObjName); - // combined ETag returned by AWS (combination of part ETags - // with number of parts at the end) - assert.strictEqual(res.ETag, - '"5bba96810ff449d94aa8f5c5a859b0cb-2"'); - }).catch(err => { - checkNoError(err); - }); + }), + ) + .then(res => { + assert.strictEqual(res.CopyPartResult.ETag, totalMpuObjectHash); + assert(res.CopyPartResult.LastModified); }); - }); }); - it('should copy two parts with range headers from a source ' + - 'bucket to a different destination bucket and ' + - 'complete the MPU', () => { - process.stdout.write('Putting first part in MPU range test\n'); - const part1ETag = '"b1e0d096c8f0670c5367d131e392b84a"'; - const part2ETag = '"a2468d5c0ec2d4d5fc13b73beb63080a"'; - // combined ETag returned by AWS (combination of part ETags - // with number of parts at the end) - const finalCombinedETag = - '"e08ede4e8b942e18537cb2289f613ae3-2"'; - return s3.send(new UploadPartCopyCommand({ - Bucket: destBucketName, - Key: destObjName, - CopySource: `${sourceBucketName}/${sourceMpuKey}`, - PartNumber: 1, - UploadId: uploadId, - CopySourceRange: 'bytes=5242890-15242880', - })).then(res => { - assert.strictEqual(res.CopyPartResult.ETag, part1ETag); - assert(res.CopyPartResult.LastModified); - }).then(() => { - process.stdout.write('Putting second part in MPU test\n'); - return s3.send(new UploadPartCopyCommand({ - Bucket: destBucketName, - Key: destObjName, - CopySource: `${sourceBucketName}/${sourceMpuKey}`, - PartNumber: 2, - UploadId: uploadId, - CopySourceRange: 'bytes=15242891-30242991', - })).then(res => { - assert.strictEqual(res.CopyPartResult.ETag, part2ETag); - assert(res.CopyPartResult.LastModified); - }).then(() => { - process.stdout.write('Completing MPU\n'); - return s3.send(new CompleteMultipartUploadCommand({ - Bucket: destBucketName, - Key: destObjName, - UploadId: uploadId, - MultipartUpload: { - Parts: [ - { ETag: part1ETag, PartNumber: 1 }, - { ETag: part2ETag, PartNumber: 2 }, - ], - }, - })).then(res => { - assert.strictEqual(res.Bucket, destBucketName); - assert.strictEqual(res.Key, destObjName); - assert.strictEqual(res.ETag, finalCombinedETag); - }).then(() => { - process.stdout.write('Getting new object\n'); - return s3.send(new GetObjectCommand({ + it( + 'should copy two parts from a source bucket to a different ' + + 'destination bucket and complete the MPU', + () => { + process.stdout.write('Putting first part in MPU test\n'); + return s3 + .send( + new UploadPartCopyCommand({ Bucket: destBucketName, Key: destObjName, - })).then(res => { - assert.strictEqual(res.ContentLength, 25000092); - assert.strictEqual(res.ETag, finalCombinedETag); - }) - .catch(err => { - checkNoError(err); - }); + CopySource: `${sourceBucketName}/${sourceMpuKey}`, + PartNumber: 1, + UploadId: uploadId, + }), + ) + .then(res => { + assert.strictEqual(res.CopyPartResult.ETag, totalMpuObjectHash); + assert(res.CopyPartResult.LastModified); + }) + .then(() => { + process.stdout.write('Putting second part in MPU test\n'); + return s3 + .send( + new UploadPartCopyCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceMpuKey}`, + PartNumber: 2, + UploadId: uploadId, + }), + ) + .then(res => { + assert.strictEqual(res.CopyPartResult.ETag, totalMpuObjectHash); + assert(res.CopyPartResult.LastModified); + }) + .then(() => { + process.stdout.write('Completing MPU\n'); + return s3 + .send( + new CompleteMultipartUploadCommand({ + Bucket: destBucketName, + Key: destObjName, + UploadId: uploadId, + MultipartUpload: { + Parts: [ + { ETag: totalMpuObjectHash, PartNumber: 1 }, + { ETag: totalMpuObjectHash, PartNumber: 2 }, + ], + }, + }), + ) + .then(res => { + assert.strictEqual(res.Bucket, destBucketName); + assert.strictEqual(res.Key, destObjName); + // combined ETag returned by AWS (combination of part ETags + // with number of parts at the end) + assert.strictEqual(res.ETag, '"5bba96810ff449d94aa8f5c5a859b0cb-2"'); + }) + .catch(err => { + checkNoError(err); + }); + }); }); - }); - }); - }); + }, + ); + + it( + 'should copy two parts with range headers from a source ' + + 'bucket to a different destination bucket and ' + + 'complete the MPU', + () => { + process.stdout.write('Putting first part in MPU range test\n'); + const part1ETag = '"b1e0d096c8f0670c5367d131e392b84a"'; + const part2ETag = '"a2468d5c0ec2d4d5fc13b73beb63080a"'; + // combined ETag returned by AWS (combination of part ETags + // with number of parts at the end) + const finalCombinedETag = '"e08ede4e8b942e18537cb2289f613ae3-2"'; + return s3 + .send( + new UploadPartCopyCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceMpuKey}`, + PartNumber: 1, + UploadId: uploadId, + CopySourceRange: 'bytes=5242890-15242880', + }), + ) + .then(res => { + assert.strictEqual(res.CopyPartResult.ETag, part1ETag); + assert(res.CopyPartResult.LastModified); + }) + .then(() => { + process.stdout.write('Putting second part in MPU test\n'); + return s3 + .send( + new UploadPartCopyCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceMpuKey}`, + PartNumber: 2, + UploadId: uploadId, + CopySourceRange: 'bytes=15242891-30242991', + }), + ) + .then(res => { + assert.strictEqual(res.CopyPartResult.ETag, part2ETag); + assert(res.CopyPartResult.LastModified); + }) + .then(() => { + process.stdout.write('Completing MPU\n'); + return s3 + .send( + new CompleteMultipartUploadCommand({ + Bucket: destBucketName, + Key: destObjName, + UploadId: uploadId, + MultipartUpload: { + Parts: [ + { ETag: part1ETag, PartNumber: 1 }, + { ETag: part2ETag, PartNumber: 2 }, + ], + }, + }), + ) + .then(res => { + assert.strictEqual(res.Bucket, destBucketName); + assert.strictEqual(res.Key, destObjName); + assert.strictEqual(res.ETag, finalCombinedETag); + }) + .then(() => { + process.stdout.write('Getting new object\n'); + return s3 + .send( + new GetObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + }), + ) + .then(res => { + assert.strictEqual(res.ContentLength, 25000092); + assert.strictEqual(res.ETag, finalCombinedETag); + }) + .catch(err => { + checkNoError(err); + }); + }); + }); + }); + }, + ); it('should overwrite an existing part by copying a part', () => { // AWS response etag for this completed MPU const finalObjETag = '"db77ebbae9e9f5a244a26b86193ad818-1"'; process.stdout.write('Putting first part in MPU test\n'); - return s3.send(new UploadPartCopyCommand({ - Bucket: destBucketName, - Key: destObjName, - CopySource: `${sourceBucketName}/${sourceMpuKey}`, - PartNumber: 1, - UploadId: uploadId, - })).then(res => { - assert.strictEqual(res.CopyPartResult.ETag, totalMpuObjectHash); - assert(res.CopyPartResult.LastModified); - }).then(() => { - process.stdout.write('Overwriting first part in MPU test\n'); - return s3.send(new UploadPartCopyCommand({ - Bucket: destBucketName, - Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - PartNumber: 1, - UploadId: uploadId - }) - ).then(res => { - assert.strictEqual(res.CopyPartResult.ETag, etag); - assert(res.CopyPartResult.LastModified); - process.stdout.write('Completing MPU\n'); - return s3.send(new CompleteMultipartUploadCommand({ - Bucket: destBucketName, - Key: destObjName, - UploadId: uploadId, - MultipartUpload: { - Parts: [ - { ETag: etag, PartNumber: 1 }, - ], - }, - }) - ).then(res => { - assert.strictEqual(res.Bucket, destBucketName); - assert.strictEqual(res.Key, destObjName); - assert.strictEqual(res.ETag, finalObjETag); - }).then(() => { - process.stdout.write('Getting object put by MPU with ' + - 'overwrite part\n'); - return s3.send(new GetObjectCommand({ + return s3 + .send( + new UploadPartCopyCommand({ Bucket: destBucketName, Key: destObjName, - })).then(res => { - assert.strictEqual(res.ETag, finalObjETag); - }).catch(err => { - checkNoError(err); - }); + CopySource: `${sourceBucketName}/${sourceMpuKey}`, + PartNumber: 1, + UploadId: uploadId, + }), + ) + .then(res => { + assert.strictEqual(res.CopyPartResult.ETag, totalMpuObjectHash); + assert(res.CopyPartResult.LastModified); + }) + .then(() => { + process.stdout.write('Overwriting first part in MPU test\n'); + return s3 + .send( + new UploadPartCopyCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + PartNumber: 1, + UploadId: uploadId, + }), + ) + .then(res => { + assert.strictEqual(res.CopyPartResult.ETag, etag); + assert(res.CopyPartResult.LastModified); + process.stdout.write('Completing MPU\n'); + return s3 + .send( + new CompleteMultipartUploadCommand({ + Bucket: destBucketName, + Key: destObjName, + UploadId: uploadId, + MultipartUpload: { + Parts: [{ ETag: etag, PartNumber: 1 }], + }, + }), + ) + .then(res => { + assert.strictEqual(res.Bucket, destBucketName); + assert.strictEqual(res.Key, destObjName); + assert.strictEqual(res.ETag, finalObjETag); + }) + .then(() => { + process.stdout.write('Getting object put by MPU with ' + 'overwrite part\n'); + return s3 + .send( + new GetObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + }), + ) + .then(res => { + assert.strictEqual(res.ETag, finalObjETag); + }) + .catch(err => { + checkNoError(err); + }); + }); + }); }); - }); }); - }); - it('should not corrupt object if overwriting an existing part by copying a part ' + - 'while the MPU is being completed', async () => { - const finalObjETag = '"db77ebbae9e9f5a244a26b86193ad818-1"'; - process.stdout.write('Putting first part in MPU test\n'); - const randomDestObjName = `copycatobject${Math.floor(Math.random() * 100000)}`; + it( + 'should not corrupt object if overwriting an existing part by copying a part ' + + 'while the MPU is being completed', + async () => { + const finalObjETag = '"db77ebbae9e9f5a244a26b86193ad818-1"'; + process.stdout.write('Putting first part in MPU test\n'); + const randomDestObjName = `copycatobject${Math.floor(Math.random() * 100000)}`; - const initiateRes = await s3.send(new CreateMultipartUploadCommand({ - Bucket: destBucketName, - Key: randomDestObjName, - })); + const initiateRes = await s3.send( + new CreateMultipartUploadCommand({ + Bucket: destBucketName, + Key: randomDestObjName, + }), + ); const uploadId = initiateRes.UploadId; - const res = await s3.send(new UploadPartCopyCommand({ - Bucket: destBucketName, - Key: randomDestObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - PartNumber: 1, - UploadId: uploadId, - })); + const res = await s3.send( + new UploadPartCopyCommand({ + Bucket: destBucketName, + Key: randomDestObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + PartNumber: 1, + UploadId: uploadId, + }), + ); assert.strictEqual(res.CopyPartResult.ETag, etag); assert(res.CopyPartResult.LastModified); - process.stdout.write( - 'Overwriting first part in MPU test and completing MPU at the same time\n', - ); + process.stdout.write('Overwriting first part in MPU test and completing MPU at the same time\n'); const [completeRes, uploadRes] = await Promise.all([ - s3.send(new CompleteMultipartUploadCommand({ - Bucket: destBucketName, - Key: randomDestObjName, - UploadId: uploadId, - MultipartUpload: { - Parts: [{ ETag: etag, PartNumber: 1 }], - }, - })).catch(err => { - throw err; - }), - s3.send(new UploadPartCopyCommand({ - Bucket: destBucketName, - Key: randomDestObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - PartNumber: 1, - UploadId: uploadId, - })) - .catch(err => { - const completeMPUFinishedEarlier = - err.name === 'NoSuchKey' || err.name === 'NoSuchUpload'; - if (completeMPUFinishedEarlier) { - return Promise.resolve(null); - } - throw err; - }), - ], - ); + s3 + .send( + new CompleteMultipartUploadCommand({ + Bucket: destBucketName, + Key: randomDestObjName, + UploadId: uploadId, + MultipartUpload: { + Parts: [{ ETag: etag, PartNumber: 1 }], + }, + }), + ) + .catch(err => { + throw err; + }), + s3 + .send( + new UploadPartCopyCommand({ + Bucket: destBucketName, + Key: randomDestObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + PartNumber: 1, + UploadId: uploadId, + }), + ) + .catch(err => { + const completeMPUFinishedEarlier = + err.name === 'NoSuchKey' || err.name === 'NoSuchUpload'; + if (completeMPUFinishedEarlier) { + return Promise.resolve(null); + } + throw err; + }), + ]); if (uploadRes !== null) { - assert.strictEqual(uploadRes.CopyPartResult.ETag, etag); - assert(uploadRes.CopyPartResult.LastModified); + assert.strictEqual(uploadRes.CopyPartResult.ETag, etag); + assert(uploadRes.CopyPartResult.LastModified); } assert.strictEqual(completeRes.Bucket, destBucketName); assert.strictEqual(completeRes.Key, randomDestObjName); assert.strictEqual(completeRes.ETag, finalObjETag); - process.stdout.write( - 'Getting object put by MPU with overwrite part\n', + process.stdout.write('Getting object put by MPU with overwrite part\n'); + const resGet = await s3.send( + new GetObjectCommand({ + Bucket: destBucketName, + Key: randomDestObjName, + }), ); - const resGet = await s3 - .send(new GetObjectCommand({ - Bucket: destBucketName, - Key: randomDestObjName, - })); assert.strictEqual(resGet.ETag, finalObjETag); - }); + }, + ); }); - it('should return an error if no such upload initiated', - () => s3.send(new UploadPartCopyCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - PartNumber: 1, - UploadId: 'madeupuploadid444233232', - })).catch(err => { + it('should return an error if no such upload initiated', () => + s3 + .send( + new UploadPartCopyCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + PartNumber: 1, + UploadId: 'madeupuploadid444233232', + }), + ) + .catch(err => { checkError(err, 'NoSuchUpload'); })); - it('should return an error if attempt to copy from nonexistent bucket', - () => s3.send(new UploadPartCopyCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: `nobucket453234/${sourceObjName}`, - PartNumber: 1, - UploadId: uploadId, - })).catch(err => { + it('should return an error if attempt to copy from nonexistent bucket', () => + s3 + .send( + new UploadPartCopyCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `nobucket453234/${sourceObjName}`, + PartNumber: 1, + UploadId: uploadId, + }), + ) + .catch(err => { checkError(err, 'NoSuchBucket'); })); - it('should return an error if attempt to copy to nonexistent bucket', - () => s3.send(new UploadPartCopyCommand({ Bucket: 'nobucket453234', Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - PartNumber: 1, - UploadId: uploadId, - })).catch(err => { + it('should return an error if attempt to copy to nonexistent bucket', () => + s3 + .send( + new UploadPartCopyCommand({ + Bucket: 'nobucket453234', + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + PartNumber: 1, + UploadId: uploadId, + }), + ) + .catch(err => { checkError(err, 'NoSuchBucket'); })); - it('should return an error if attempt to copy nonexistent object', - () => s3.send(new UploadPartCopyCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: `${sourceBucketName}/nokey`, - PartNumber: 1, - UploadId: uploadId, - })).catch(err => { + it('should return an error if attempt to copy nonexistent object', () => + s3 + .send( + new UploadPartCopyCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/nokey`, + PartNumber: 1, + UploadId: uploadId, + }), + ) + .catch(err => { checkError(err, 'NoSuchKey'); })); - it('should return an error if use invalid part number', - () => s3.send(new UploadPartCopyCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: `${sourceBucketName}/nokey`, - PartNumber: 10001, - UploadId: uploadId, - })).catch(err => { + it('should return an error if use invalid part number', () => + s3 + .send( + new UploadPartCopyCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/nokey`, + PartNumber: 10001, + UploadId: uploadId, + }), + ) + .catch(err => { checkError(err, 'InvalidArgument'); })); @@ -683,45 +907,51 @@ describe('Object Part Copy', () => { const archive = { archiveInfo: { archiveId: '97a71dfe-49c1-4cca-840a-69199e0b0322', - archiveVersion: 5577006791947779 + archiveVersion: 5577006791947779, }, }; fakeMetadataArchive(sourceBucketName, sourceObjName, undefined, archive, err => { assert.ifError(err); - s3.send(new UploadPartCopyCommand({ - Bucket: destBucketName, - Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - PartNumber: 1, - UploadId: uploadId, - })).then(() => { - done(new Error('Expected failure but got success')); - }).catch(err => { - + s3.send( + new UploadPartCopyCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + PartNumber: 1, + UploadId: uploadId, + }), + ) + .then(() => { + done(new Error('Expected failure but got success')); + }) + .catch(err => { assert.strictEqual(err.$metadata.httpStatusCode, 403); done(); }); }); }); - it('should copy a part of an object when it\'s transitioning to cold', done => { + it("should copy a part of an object when it's transitioning to cold", done => { fakeMetadataTransition(sourceBucketName, sourceObjName, undefined, err => { assert.ifError(err); - s3.send(new UploadPartCopyCommand({ - Bucket: destBucketName, - Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - PartNumber: 1, - UploadId: uploadId, - })).then(res => { - assert.strictEqual(res.CopyPartResult.ETag, etag); - assert(res.CopyPartResult.LastModified); - done(); - }).catch(err => { - checkNoError(err); - done(err); - }); - + s3.send( + new UploadPartCopyCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + PartNumber: 1, + UploadId: uploadId, + }), + ) + .then(res => { + assert.strictEqual(res.CopyPartResult.ETag, etag); + assert(res.CopyPartResult.LastModified); + done(); + }) + .catch(err => { + checkNoError(err); + done(err); + }); }); }); @@ -731,24 +961,28 @@ describe('Object Part Copy', () => { restoreRequestedAt: new Date(0), restoreRequestedDays: 5, restoreCompletedAt: new Date(10), - restoreWillExpireAt: new Date(10 + (5 * 24 * 60 * 60 * 1000)), + restoreWillExpireAt: new Date(10 + 5 * 24 * 60 * 60 * 1000), }; fakeMetadataArchive(sourceBucketName, sourceObjName, undefined, archiveCompleted, err => { assert.ifError(err); - s3.send(new UploadPartCopyCommand({ - Bucket: destBucketName, - Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - PartNumber: 1, - UploadId: uploadId, - })).then(res => { - assert.strictEqual(res.CopyPartResult.ETag, etag); - assert(res.CopyPartResult.LastModified); - done(); - }).catch(err => { - checkNoError(err); - done(err); - }); + s3.send( + new UploadPartCopyCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + PartNumber: 1, + UploadId: uploadId, + }), + ) + .then(res => { + assert.strictEqual(res.CopyPartResult.ETag, etag); + assert(res.CopyPartResult.LastModified); + done(); + }) + .catch(err => { + checkNoError(err); + done(err); + }); }); }); }); @@ -760,82 +994,122 @@ describe('Object Part Copy', () => { beforeEach(() => { process.stdout.write('In other account before each\n'); - return otherAccountS3.send(new CreateBucketCommand({ Bucket: - otherAccountBucket })) - .catch(err => { - process.stdout.write('Error creating other account ' + - `bucket: ${err}\n`); - throw err; - }).then(() => { - process.stdout.write('Initiating other account MPU\n'); - return otherAccountS3.send(new CreateMultipartUploadCommand({ - Bucket: otherAccountBucket, - Key: otherAccountKey, - })); - }).then(initiateRes => { - otherAccountUploadId = initiateRes.UploadId; - }).catch(err => { - process.stdout.write('Error in other account ' + - `beforeEach: ${err}\n`); - throw err; - }); + return otherAccountS3 + .send(new CreateBucketCommand({ Bucket: otherAccountBucket })) + .catch(err => { + process.stdout.write('Error creating other account ' + `bucket: ${err}\n`); + throw err; + }) + .then(() => { + process.stdout.write('Initiating other account MPU\n'); + return otherAccountS3.send( + new CreateMultipartUploadCommand({ + Bucket: otherAccountBucket, + Key: otherAccountKey, + }), + ); + }) + .then(initiateRes => { + otherAccountUploadId = initiateRes.UploadId; + }) + .catch(err => { + process.stdout.write('Error in other account ' + `beforeEach: ${err}\n`); + throw err; + }); }); - afterEach(() => otherAccountBucketUtility.empty(otherAccountBucket) - .then(() => otherAccountS3.send(new AbortMultipartUploadCommand({ - Bucket: otherAccountBucket, - Key: otherAccountKey, - UploadId: otherAccountUploadId, - }))) - .catch(err => { - if (err.name !== 'NoSuchUpload') { - process.stdout.write('Error in other account ' + - `afterEach: ${err}\n`); - throw err; - } - }).then(() => { - otherAccountBucketUtility.deleteOne(otherAccountBucket); - }) + afterEach(() => + otherAccountBucketUtility + .empty(otherAccountBucket) + .then(() => + otherAccountS3.send( + new AbortMultipartUploadCommand({ + Bucket: otherAccountBucket, + Key: otherAccountKey, + UploadId: otherAccountUploadId, + }), + ), + ) + .catch(err => { + if (err.name !== 'NoSuchUpload') { + process.stdout.write('Error in other account ' + `afterEach: ${err}\n`); + throw err; + } + }) + .then(() => { + otherAccountBucketUtility.deleteOne(otherAccountBucket); + }), ); - it('should not allow an account without read persmission on the ' + - 'source object to copy the object', () => otherAccountS3.send(new UploadPartCopyCommand( - { Bucket: otherAccountBucket, - Key: otherAccountKey, - CopySource: `${sourceBucketName}/${sourceObjName}`, - PartNumber: 1, - UploadId: otherAccountUploadId, - })).catch( - err => { - checkError(err, 'AccessDenied'); - })); - - it('should not allow an account without write persmission on the ' + - 'destination bucket to upload part copy the object', () => { - otherAccountS3.send(new PutObjectCommand({ Bucket: otherAccountBucket, - Key: otherAccountKey, Body: '' })).then(() => otherAccountS3.send( - new UploadPartCopyCommand({ Bucket: destBucketName, - Key: destObjName, - CopySource: `${otherAccountBucket}/${otherAccountKey}`, - PartNumber: 1, - UploadId: uploadId, - })).catch(err => checkError(err, 'AccessDenied'))); - }); + it( + 'should not allow an account without read persmission on the ' + 'source object to copy the object', + () => + otherAccountS3 + .send( + new UploadPartCopyCommand({ + Bucket: otherAccountBucket, + Key: otherAccountKey, + CopySource: `${sourceBucketName}/${sourceObjName}`, + PartNumber: 1, + UploadId: otherAccountUploadId, + }), + ) + .catch(err => { + checkError(err, 'AccessDenied'); + }), + ); - it('should allow an account with read permission on the ' + - 'source object and write permission on the destination ' + - 'bucket to upload part copy the object', () => s3.send(new PutObjectAclCommand( - { Bucket: sourceBucketName, - Key: sourceObjName, ACL: 'public-read' })).then(() => otherAccountS3.send(new UploadPartCopyCommand( - { Bucket: otherAccountBucket, - Key: otherAccountKey, - CopySource: `${sourceBucketName}/${sourceObjName}`, - PartNumber: 1, - UploadId: otherAccountUploadId, - })).catch(err => { - checkNoError(err); - } - ))); + it( + 'should not allow an account without write persmission on the ' + + 'destination bucket to upload part copy the object', + () => { + otherAccountS3 + .send(new PutObjectCommand({ Bucket: otherAccountBucket, Key: otherAccountKey, Body: '' })) + .then(() => + otherAccountS3 + .send( + new UploadPartCopyCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${otherAccountBucket}/${otherAccountKey}`, + PartNumber: 1, + UploadId: uploadId, + }), + ) + .catch(err => checkError(err, 'AccessDenied')), + ); + }, + ); + + it( + 'should allow an account with read permission on the ' + + 'source object and write permission on the destination ' + + 'bucket to upload part copy the object', + () => + s3 + .send( + new PutObjectAclCommand({ + Bucket: sourceBucketName, + Key: sourceObjName, + ACL: 'public-read', + }), + ) + .then(() => + otherAccountS3 + .send( + new UploadPartCopyCommand({ + Bucket: otherAccountBucket, + Key: otherAccountKey, + CopySource: `${sourceBucketName}/${sourceObjName}`, + PartNumber: 1, + UploadId: otherAccountUploadId, + }), + ) + .catch(err => { + checkNoError(err); + }), + ), + ); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/object/corsErrorHeaders.js b/tests/functional/aws-node-sdk/test/object/corsErrorHeaders.js index 9621c84e8c..1ee4c1b722 100644 --- a/tests/functional/aws-node-sdk/test/object/corsErrorHeaders.js +++ b/tests/functional/aws-node-sdk/test/object/corsErrorHeaders.js @@ -8,8 +8,7 @@ const { const assert = require('assert'); const getConfig = require('../support/config'); -const { methodRequest, generateCorsParams } = - require('../../lib/utility/cors-util'); +const { methodRequest, generateCorsParams } = require('../../lib/utility/cors-util'); const config = getConfig('default', { signatureVersion: 'v4' }); const s3 = new S3Client({ ...config, forcePathStyle: true }); @@ -17,8 +16,7 @@ const s3 = new S3Client({ ...config, forcePathStyle: true }); const bucket = 'corserrorheadertest'; const objectKey = 'objectKey'; const allowedOrigin = 'http://www.allowed.test'; -const vary = 'Origin, Access-Control-Request-Headers, ' - + 'Access-Control-Request-Method'; +const vary = 'Origin, Access-Control-Request-Headers, ' + 'Access-Control-Request-Method'; const expectedCorsHeaders = { 'access-control-allow-origin': allowedOrigin, @@ -36,32 +34,19 @@ const corsParams = generateCorsParams(bucket, { // Raw unauthenticated requests - they always return 403. // Each spec describes (method, path, query) against the bucket. const unauthenticatedRequests = [ - { description: 'GET bucket (list objects)', - method: 'GET', query: null, objectKey: null }, - { description: 'HEAD bucket', - method: 'HEAD', query: null, objectKey: null }, - { description: 'DELETE bucket', - method: 'DELETE', query: null, objectKey: null }, - { description: 'GET bucket ACL', - method: 'GET', query: 'acl', objectKey: null }, - { description: 'GET bucket CORS', - method: 'GET', query: 'cors', objectKey: null }, - { description: 'GET bucket versioning', - method: 'GET', query: 'versioning', objectKey: null }, - { description: 'GET bucket website', - method: 'GET', query: 'website', objectKey: null }, - { description: 'GET bucket tagging', - method: 'GET', query: 'tagging', objectKey: null }, - { description: 'GET object', - method: 'GET', query: null, objectKey }, - { description: 'HEAD object', - method: 'HEAD', query: null, objectKey }, - { description: 'PUT object', - method: 'PUT', query: null, objectKey }, - { description: 'DELETE object', - method: 'DELETE', query: null, objectKey }, - { description: 'GET bucket uploads (list multipart uploads)', - method: 'GET', query: 'uploads', objectKey: null }, + { description: 'GET bucket (list objects)', method: 'GET', query: null, objectKey: null }, + { description: 'HEAD bucket', method: 'HEAD', query: null, objectKey: null }, + { description: 'DELETE bucket', method: 'DELETE', query: null, objectKey: null }, + { description: 'GET bucket ACL', method: 'GET', query: 'acl', objectKey: null }, + { description: 'GET bucket CORS', method: 'GET', query: 'cors', objectKey: null }, + { description: 'GET bucket versioning', method: 'GET', query: 'versioning', objectKey: null }, + { description: 'GET bucket website', method: 'GET', query: 'website', objectKey: null }, + { description: 'GET bucket tagging', method: 'GET', query: 'tagging', objectKey: null }, + { description: 'GET object', method: 'GET', query: null, objectKey }, + { description: 'HEAD object', method: 'HEAD', query: null, objectKey }, + { description: 'PUT object', method: 'PUT', query: null, objectKey }, + { description: 'DELETE object', method: 'DELETE', query: null, objectKey }, + { description: 'GET bucket uploads (list multipart uploads)', method: 'GET', query: 'uploads', objectKey: null }, // GET bucket policy and POST multi-delete are not covered here: the // first returns 405 (method rejected pre-auth), the second returns 400 // (missing XML body fails validation pre-auth). Neither reaches the @@ -80,27 +65,29 @@ describe('CORS headers on 403 responses when bucket has CORS configured', () => }); unauthenticatedRequests.forEach(spec => { - it(`returns CORS headers on 403 for ${spec.description} ` - + 'when Origin matches a rule', done => { - methodRequest({ - method: spec.method, - bucket, - objectKey: spec.objectKey, - query: spec.query, - headers: { origin: allowedOrigin }, - // Use numeric status: HEAD responses have no body, and some - // endpoints (bucket policy, multi-delete) can fail with a - // non-AccessDenied body before auth even runs. We only care - // about the 403 status and the CORS headers here. - code: 403, - headersResponse: expectedCorsHeaders, - }, done); + it(`returns CORS headers on 403 for ${spec.description} ` + 'when Origin matches a rule', done => { + methodRequest( + { + method: spec.method, + bucket, + objectKey: spec.objectKey, + query: spec.query, + headers: { origin: allowedOrigin }, + // Use numeric status: HEAD responses have no body, and some + // endpoints (bucket policy, multi-delete) can fail with a + // non-AccessDenied body before auth even runs. We only care + // about the 403 status and the CORS headers here. + code: 403, + headersResponse: expectedCorsHeaders, + }, + done, + ); }); }); - it('omits CORS headers on 403 when Origin does not match any rule', - done => { - methodRequest({ + it('omits CORS headers on 403 when Origin does not match any rule', done => { + methodRequest( + { method: 'GET', bucket, query: null, @@ -109,20 +96,24 @@ describe('CORS headers on 403 responses when bucket has CORS configured', () => code: 403, // headersResponse unset -> cors-util asserts CORS headers // are NOT present. - }, done); - }); + }, + done, + ); + }); - it('omits CORS headers on 403 when no Origin header is sent', - done => { - methodRequest({ + it('omits CORS headers on 403 when no Origin header is sent', done => { + methodRequest( + { method: 'GET', bucket, query: null, objectKey: null, headers: {}, code: 403, - }, done); - }); + }, + done, + ); + }); }); describe('CORS headers on 200 responses (regression guard)', () => { @@ -135,26 +126,30 @@ describe('CORS headers on 200 responses (regression guard)', () => { await s3.send(new DeleteBucketCommand({ Bucket: bucket })); }); - it('returns CORS headers on a successful list objects (200)', - async () => { - const command = new ListObjectsCommand({ Bucket: bucket }); - // Inject Origin on the outgoing request and capture the raw - // response headers via the deserialize step. - command.middlewareStack.add(next => async args => { + it('returns CORS headers on a successful list objects (200)', async () => { + const command = new ListObjectsCommand({ Bucket: bucket }); + // Inject Origin on the outgoing request and capture the raw + // response headers via the deserialize step. + command.middlewareStack.add( + next => async args => { const headers = args.request && args.request.headers; if (headers) { headers.origin = allowedOrigin; } return next(args); - }, { step: 'build' }); - let responseHeaders; - command.middlewareStack.add(next => async args => { + }, + { step: 'build' }, + ); + let responseHeaders; + command.middlewareStack.add( + next => async args => { const result = await next(args); responseHeaders = result.response && result.response.headers; return result; - }, { step: 'deserialize' }); - await s3.send(command); - assert.strictEqual(responseHeaders['access-control-allow-origin'], - allowedOrigin); - }); + }, + { step: 'deserialize' }, + ); + await s3.send(command); + assert.strictEqual(responseHeaders['access-control-allow-origin'], allowedOrigin); + }); }); diff --git a/tests/functional/aws-node-sdk/test/object/corsHeaders.js b/tests/functional/aws-node-sdk/test/object/corsHeaders.js index 45d0e1a209..4da1f2dc22 100644 --- a/tests/functional/aws-node-sdk/test/object/corsHeaders.js +++ b/tests/functional/aws-node-sdk/test/object/corsHeaders.js @@ -1,35 +1,37 @@ -const { S3Client, - ListObjectsCommand, - GetBucketAclCommand, - GetBucketCorsCommand, - GetBucketVersioningCommand, - GetBucketLocationCommand, - GetBucketWebsiteCommand, - ListMultipartUploadsCommand, - GetObjectCommand, - GetObjectAclCommand, - ListPartsCommand, - HeadBucketCommand, - HeadObjectCommand, - CreateBucketCommand, - PutBucketAclCommand, - PutBucketVersioningCommand, - PutBucketWebsiteCommand, - PutBucketCorsCommand, - PutObjectCommand, - PutObjectAclCommand, - CopyObjectCommand, - UploadPartCommand, - UploadPartCopyCommand, - CreateMultipartUploadCommand, - CompleteMultipartUploadCommand, - DeleteObjectsCommand, - DeleteBucketCommand, - DeleteBucketWebsiteCommand, - DeleteBucketCorsCommand, - DeleteObjectCommand, - AbortMultipartUploadCommand, - ListBucketsCommand } = require('@aws-sdk/client-s3'); +const { + S3Client, + ListObjectsCommand, + GetBucketAclCommand, + GetBucketCorsCommand, + GetBucketVersioningCommand, + GetBucketLocationCommand, + GetBucketWebsiteCommand, + ListMultipartUploadsCommand, + GetObjectCommand, + GetObjectAclCommand, + ListPartsCommand, + HeadBucketCommand, + HeadObjectCommand, + CreateBucketCommand, + PutBucketAclCommand, + PutBucketVersioningCommand, + PutBucketWebsiteCommand, + PutBucketCorsCommand, + PutObjectCommand, + PutObjectAclCommand, + CopyObjectCommand, + UploadPartCommand, + UploadPartCopyCommand, + CreateMultipartUploadCommand, + CompleteMultipartUploadCommand, + DeleteObjectsCommand, + DeleteBucketCommand, + DeleteBucketWebsiteCommand, + DeleteBucketCorsCommand, + DeleteObjectCommand, + AbortMultipartUploadCommand, + ListBucketsCommand, +} = require('@aws-sdk/client-s3'); const { promisify } = require('util'); const assert = require('assert'); @@ -48,8 +50,7 @@ const bucket = 'bucketcorsheadertest'; const objectKey = 'objectKeyName'; const allowedOrigin = 'http://www.allowedwebsite.com'; const notAllowedOrigin = 'http://www.notallowedwebsite.com'; -const vary = 'Origin, Access-Control-Request-Headers, ' + - 'Access-Control-Request-Method'; +const vary = 'Origin, Access-Control-Request-Headers, ' + 'Access-Control-Request-Method'; const defaultOptions = { allowedMethods: ['GET'], allowedOrigins: [allowedOrigin], @@ -152,10 +153,12 @@ const apiMethods = [ params: { Bucket: bucket, CORSConfiguration: { - CORSRules: [{ - AllowedOrigins: [allowedOrigin], - AllowedMethods: ['PUT'], - }], + CORSRules: [ + { + AllowedOrigins: [allowedOrigin], + AllowedMethods: ['PUT'], + }, + ], }, }, }, @@ -219,9 +222,7 @@ const apiMethods = [ params: { Bucket: bucket, Delete: { - Objects: [ - { Key: objectKey }, - ], + Objects: [{ Key: objectKey }], }, }, }, @@ -270,10 +271,12 @@ async function _checkHeaders(action, params, origin, expectedHeaders) { }); } else { // if no expectedHeaders provided, should not have these headers in the response - ['access-control-allow-origin', - 'access-control-allow-methods', - 'access-control-allow-credentials', - 'vary'].forEach(key => { + [ + 'access-control-allow-origin', + 'access-control-allow-methods', + 'access-control-allow-credentials', + 'vary', + ].forEach(key => { assert.strictEqual(resHeaders[key], undefined, `Error: ${key} should not have value`); }); } @@ -319,8 +322,8 @@ async function _checkHeaders(action, params, origin, expectedHeaders) { { step: 'finalizeRequest', name: 'captureHeaders', - priority: 'high' - } + priority: 'high', + }, ); try { @@ -330,15 +333,16 @@ async function _checkHeaders(action, params, origin, expectedHeaders) { // Clean up multipart upload if needed (equivalent to the original cleanup logic) if (response.UploadId) { - await testS3.send(new AbortMultipartUploadCommand({ - Bucket: bucket, - Key: objectKey, - UploadId: response.UploadId - })); + await testS3.send( + new AbortMultipartUploadCommand({ + Bucket: bucket, + Key: objectKey, + UploadId: response.UploadId, + }), + ); } _runAssertions(capturedHeaders); - } catch { // CORS headers should still be sent in case of errors as long as // request matches CORS configuration @@ -348,15 +352,17 @@ async function _checkHeaders(action, params, origin, expectedHeaders) { describe('Cross Origin Resource Sharing requests', () => { beforeEach(done => { - s3.send(new CreateBucketCommand({ - Bucket: bucket, - ACL: 'public-read-write' - })) - .then(() => _waitForAWS(done)) - .catch(err => { - process.stdout.write(`Error in beforeEach: ${err}\n`); - _waitForAWS(done, err); - }); + s3.send( + new CreateBucketCommand({ + Bucket: bucket, + ACL: 'public-read-write', + }), + ) + .then(() => _waitForAWS(done)) + .catch(err => { + process.stdout.write(`Error in beforeEach: ${err}\n`); + _waitForAWS(done, err); + }); }); afterEach(done => { @@ -372,29 +378,28 @@ describe('Cross Origin Resource Sharing requests', () => { }); describe('on non-existing bucket', () => { - it('should not respond to request with CORS headers, even if request was sent with Origin header', - async () => { + it('should not respond to request with CORS headers, even if request was sent with Origin header', async () => { await _checkHeaders(ListObjectsCommand, { Bucket: 'nonexistingbucket' }, allowedOrigin, null); }); }); describe('on bucket without CORS configuration', () => { - it('should not respond to request with CORS headers,' + - ' even if request was sent with Origin header', async () => { - await _checkHeaders(ListObjectsCommand, { Bucket: bucket }, allowedOrigin, null); - }); + it( + 'should not respond to request with CORS headers,' + ' even if request was sent with Origin header', + async () => { + await _checkHeaders(ListObjectsCommand, { Bucket: bucket }, allowedOrigin, null); + }, + ); }); - describe('on bucket with CORS configuration: ' + - 'allow one origin and all methods', () => { + describe('on bucket with CORS configuration: ' + 'allow one origin and all methods', () => { const corsParams = generateCorsParams(bucket, { allowedMethods: ['GET', 'PUT', 'HEAD', 'POST', 'DELETE'], allowedOrigins: [allowedOrigin], }); const expectedHeaders = { 'access-control-allow-origin': allowedOrigin, - 'access-control-allow-methods': corsParams.CORSConfiguration - .CORSRules[0].AllowedMethods.join(', '), + 'access-control-allow-methods': corsParams.CORSConfiguration.CORSRules[0].AllowedMethods.join(', '), 'access-control-allow-credentials': 'true', vary, }; @@ -405,8 +410,7 @@ describe('Cross Origin Resource Sharing requests', () => { afterEach(done => { removeAllVersions({ Bucket: bucket }, err => { - if (err && err.name !== 'NoSuchKey' && - err.name !== 'NoSuchBucket') { + if (err && err.name !== 'NoSuchKey' && err.name !== 'NoSuchBucket') { process.stdout.write(`Unexpected err in afterEach: ${err}`); return done(err); } @@ -415,16 +419,22 @@ describe('Cross Origin Resource Sharing requests', () => { }); describe('when request Origin/method match CORS configuration', () => { - it('should not respond with CORS headers to GET service (list buckets), ' + - 'even if Origin/method match CORS rule', async () => { - await _checkHeaders(ListBucketsCommand, {}, allowedOrigin, null); - }); - - it('should not respond with CORS headers after deleting bucket, ' + - 'even if Origin/method match CORS rule', async () => { - await s3.send(new DeleteBucketCommand({ Bucket: bucket })); - await _checkHeaders(ListObjectsCommand, { Bucket: bucket }, allowedOrigin, null); - }); + it( + 'should not respond with CORS headers to GET service (list buckets), ' + + 'even if Origin/method match CORS rule', + async () => { + await _checkHeaders(ListBucketsCommand, {}, allowedOrigin, null); + }, + ); + + it( + 'should not respond with CORS headers after deleting bucket, ' + + 'even if Origin/method match CORS rule', + async () => { + await s3.send(new DeleteBucketCommand({ Bucket: bucket })); + await _checkHeaders(ListObjectsCommand, { Bucket: bucket }, allowedOrigin, null); + }, + ); apiMethods.forEach(method => { it(`should respond to ${method.description} with CORS headers (access-control-allow-origin, @@ -462,10 +472,8 @@ describe('Cross Origin Resource Sharing requests', () => { }); }); - describe('on bucket with CORS configuration and website configuration', - () => { - const bucket = process.env.AWS_ON_AIR ? 'awsbucketwebsitetester' : - 'bucketwebsitetester'; + describe('on bucket with CORS configuration and website configuration', () => { + const bucket = process.env.AWS_ON_AIR ? 'awsbucketwebsitetester' : 'bucketwebsitetester'; const corsParams = generateCorsParams(bucket, { allowedMethods: ['GET', 'HEAD'], allowedOrigins: [allowedOrigin], @@ -485,46 +493,67 @@ describe('Cross Origin Resource Sharing requests', () => { await s3.send(new CreateBucketCommand({ Bucket: bucket, ACL: 'public-read' })); await s3.send(new PutBucketCorsCommand(corsParams)); await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })); - await s3.send(new PutObjectCommand({ Bucket: bucket, Key: 'index.html', - ACL: 'public-read', - Body: 'test content' })); + await s3.send( + new PutObjectCommand({ Bucket: bucket, Key: 'index.html', ACL: 'public-read', Body: 'test content' }), + ); }); afterEach(done => { - s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: 'index.html' - })) - .then(() => s3.send(new DeleteBucketCommand({ Bucket: bucket }))) - .then(() => _waitForAWS(done)) - .catch(err => { - process.stdout.write(`Error in afterEach: ${err}\n`); - _waitForAWS(done, err); - }); + s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: 'index.html', + }), + ) + .then(() => s3.send(new DeleteBucketCommand({ Bucket: bucket }))) + .then(() => _waitForAWS(done)) + .catch(err => { + process.stdout.write(`Error in afterEach: ${err}\n`); + _waitForAWS(done, err); + }); }); it('should respond with CORS headers at website endpoint (GET)', async () => { const headers = { Origin: allowedOrigin }; - await methodRequestPromise({ method: 'GET', bucket, - headers, headersResponse, code: 200, isWebsite: true }); + await methodRequestPromise({ method: 'GET', bucket, headers, headersResponse, code: 200, isWebsite: true }); }); it('should respond with CORS headers at website endpoint (GET) even in case of error', async () => { const headers = { Origin: allowedOrigin }; - await methodRequestPromise({ method: 'GET', bucket, objectKey: 'test', - headers, headersResponse, code: 404, isWebsite: true }); + await methodRequestPromise({ + method: 'GET', + bucket, + objectKey: 'test', + headers, + headersResponse, + code: 404, + isWebsite: true, + }); }); it('should respond with CORS headers at website endpoint (GET) even in case of redirect', async () => { const headers = { Origin: allowedOrigin }; - await methodRequestPromise({ method: 'GET', bucket, objectKey: 'redirect', - headers, headersResponse, code: 301, isWebsite: true }); + await methodRequestPromise({ + method: 'GET', + bucket, + objectKey: 'redirect', + headers, + headersResponse, + code: 301, + isWebsite: true, + }); }); it('should respond with CORS headers at website endpoint (HEAD)', async () => { const headers = { Origin: allowedOrigin }; - await methodRequestPromise({ method: 'HEAD', bucket, headers, headersResponse, - code: 200, isWebsite: true }); + await methodRequestPromise({ + method: 'HEAD', + bucket, + headers, + headersResponse, + code: 200, + isWebsite: true, + }); }); }); @@ -548,27 +577,38 @@ describe('Cross Origin Resource Sharing requests', () => { await s3.send(new PutBucketCorsCommand(corsParams)); }); - it('should not return access-control-allow-headers response header ' + - 'even if request matches CORS rule and other access-control headers are returned', async () => { - const headers = { - 'Origin': allowedOrigin, - 'Content-Type': 'testvalue', - }; - const headersOmitted = ['access-control-allow-headers']; - await methodRequestPromise({ method: 'GET', bucket, headers, headersResponse, - headersOmitted, code: 200 }); - }); - - it('Request with matching Origin/method but additional headers that violate CORS rule:\n\t should still ' + - 'respond with access-control headers (headers are only checked in preflight requests)', async () => { - const headers = { - Origin: allowedOrigin, - Test: 'test', - Expires: 86400, - }; - await methodRequestPromise({ method: 'GET', bucket, headers, - headersResponse, code: 200 }); - }); + it( + 'should not return access-control-allow-headers response header ' + + 'even if request matches CORS rule and other access-control headers are returned', + async () => { + const headers = { + Origin: allowedOrigin, + 'Content-Type': 'testvalue', + }; + const headersOmitted = ['access-control-allow-headers']; + await methodRequestPromise({ + method: 'GET', + bucket, + headers, + headersResponse, + headersOmitted, + code: 200, + }); + }, + ); + + it( + 'Request with matching Origin/method but additional headers that violate CORS rule:\n\t should still ' + + 'respond with access-control headers (headers are only checked in preflight requests)', + async () => { + const headers = { + Origin: allowedOrigin, + Test: 'test', + Expires: 86400, + }; + await methodRequestPromise({ method: 'GET', bucket, headers, headersResponse, code: 200 }); + }, + ); }); [ diff --git a/tests/functional/aws-node-sdk/test/object/corsPreflight.js b/tests/functional/aws-node-sdk/test/object/corsPreflight.js index 41109f6d3c..7a79179c8f 100644 --- a/tests/functional/aws-node-sdk/test/object/corsPreflight.js +++ b/tests/functional/aws-node-sdk/test/object/corsPreflight.js @@ -17,14 +17,9 @@ const s3 = new S3Client(config); const bucket = 'bucketcorstester'; const methods = ['PUT', 'POST', 'DELETE', 'GET']; -const originsWithWildcards = [ - '*.allowedorigin.com', - 'http://*.allowedorigin.com', - 'http://www.allowedorigin.*', -]; +const originsWithWildcards = ['*.allowedorigin.com', 'http://*.allowedorigin.com', 'http://www.allowedorigin.*']; const allowedOrigin = 'http://www.allowedwebsite.com'; -const vary = 'Origin, Access-Control-Request-Headers, ' + - 'Access-Control-Request-Method'; +const vary = 'Origin, Access-Control-Request-Headers, ' + 'Access-Control-Request-Method'; // AWS seems to take a bit long so sometimes by the time we send the request // the bucket has not yet been created or the bucket has been deleted. @@ -41,21 +36,17 @@ describe('Preflight CORS request on non-existing bucket', () => { const headers = { Origin: allowedOrigin, }; - methodRequest({ method: 'GET', bucket, headers, code: 'NoSuchBucket', - headersResponse: null }, done); + methodRequest({ method: 'GET', bucket, headers, code: 'NoSuchBucket', headersResponse: null }, done); }); it('should return BadRequest for OPTIONS request without origin', done => { const headers = {}; - methodRequest({ method: 'OPTIONS', bucket, headers, code: 'BadRequest', - headersResponse: null }, done); + methodRequest({ method: 'OPTIONS', bucket, headers, code: 'BadRequest', headersResponse: null }, done); }); - it('should return BadRequest for OPTIONS request without ' + - 'Access-Control-Request-Method', done => { + it('should return BadRequest for OPTIONS request without ' + 'Access-Control-Request-Method', done => { const headers = { Origin: allowedOrigin, }; - methodRequest({ method: 'OPTIONS', bucket, headers, code: 'BadRequest', - headersResponse: null }, done); + methodRequest({ method: 'OPTIONS', bucket, headers, code: 'BadRequest', headersResponse: null }, done); }); }); @@ -71,45 +62,34 @@ describe('Preflight CORS request with existing bucket', () => { .catch(err => _waitForAWS(done, err)); }); - it('should allow GET on bucket without cors configuration even if ' + - 'Origin header sent', done => { + it('should allow GET on bucket without cors configuration even if ' + 'Origin header sent', done => { const headers = { Origin: allowedOrigin, }; - methodRequest({ method: 'GET', bucket, headers, code: 200, - headersResponse: null }, done); + methodRequest({ method: 'GET', bucket, headers, code: 200, headersResponse: null }, done); }); - it('should allow HEAD on bucket without cors configuration even if ' + - 'Origin header sent', done => { + it('should allow HEAD on bucket without cors configuration even if ' + 'Origin header sent', done => { const headers = { Origin: allowedOrigin, }; - methodRequest({ method: 'HEAD', bucket, headers, code: 200, - headersResponse: null }, done); + methodRequest({ method: 'HEAD', bucket, headers, code: 200, headersResponse: null }, done); }); - it('should respond AccessForbidden for OPTIONS request on bucket without ' + - 'CORSConfiguration', done => { + it('should respond AccessForbidden for OPTIONS request on bucket without ' + 'CORSConfiguration', done => { const headers = { - 'Origin': allowedOrigin, + Origin: allowedOrigin, 'Access-Control-Request-Method': 'GET', }; - methodRequest({ method: 'OPTIONS', bucket, headers, - code: 'AccessForbidden', headersResponse: null }, done); + methodRequest({ method: 'OPTIONS', bucket, headers, code: 'AccessForbidden', headersResponse: null }, done); }); - describe('allow PUT, POST, DELETE, GET methods and allow only ' + - 'one origin', () => { + describe('allow PUT, POST, DELETE, GET methods and allow only ' + 'one origin', () => { const corsParams = { Bucket: bucket, CORSConfiguration: { CORSRules: [ { - AllowedMethods: [ - 'PUT', 'POST', 'DELETE', 'GET', - ], - AllowedOrigins: [ - allowedOrigin, - ], + AllowedMethods: ['PUT', 'POST', 'DELETE', 'GET'], + AllowedOrigins: [allowedOrigin], }, ], }, @@ -127,43 +107,46 @@ describe('Preflight CORS request with existing bucket', () => { }); methods.forEach(method => { - it('should respond with 200 and access control headers to ' + - 'OPTIONS request from allowed origin and allowed method ' + - `"${method}"`, done => { - const headers = { - 'Origin': allowedOrigin, - 'Access-Control-Request-Method': method, - }; - const headersResponse = { - 'access-control-allow-origin': allowedOrigin, - 'access-control-allow-methods': 'PUT, POST, DELETE, GET', - 'access-control-allow-credentials': 'true', - vary, - }; - methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, - headersResponse }, done); - }); - }); - it('should respond AccessForbidden to OPTIONS request from ' + - 'not allowed origin', done => { - const headers = { - 'Origin': allowedOrigin, - 'Access-Control-Request-Method': 'GET', - 'Access-Control-Request-Headers': 'Origin, Accept, ' + - 'Content-Type', - }; - methodRequest({ method: 'OPTIONS', bucket, headers, - code: 'AccessForbidden', headersResponse: null }, done); + it( + 'should respond with 200 and access control headers to ' + + 'OPTIONS request from allowed origin and allowed method ' + + `"${method}"`, + done => { + const headers = { + Origin: allowedOrigin, + 'Access-Control-Request-Method': method, + }; + const headersResponse = { + 'access-control-allow-origin': allowedOrigin, + 'access-control-allow-methods': 'PUT, POST, DELETE, GET', + 'access-control-allow-credentials': 'true', + vary, + }; + methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, headersResponse }, done); + }, + ); }); - it('should respond AccessForbidden to OPTIONS request with ' + - 'not allowed Access-Control-Request-Headers', done => { + it('should respond AccessForbidden to OPTIONS request from ' + 'not allowed origin', done => { const headers = { - 'Origin': 'http://www.forbiddenwebsite.com', + Origin: allowedOrigin, 'Access-Control-Request-Method': 'GET', + 'Access-Control-Request-Headers': 'Origin, Accept, ' + 'Content-Type', }; - methodRequest({ method: 'OPTIONS', bucket, headers, - code: 'AccessForbidden', headersResponse: null }, done); + methodRequest({ method: 'OPTIONS', bucket, headers, code: 'AccessForbidden', headersResponse: null }, done); }); + it( + 'should respond AccessForbidden to OPTIONS request with ' + 'not allowed Access-Control-Request-Headers', + done => { + const headers = { + Origin: 'http://www.forbiddenwebsite.com', + 'Access-Control-Request-Method': 'GET', + }; + methodRequest( + { method: 'OPTIONS', bucket, headers, code: 'AccessForbidden', headersResponse: null }, + done, + ); + }, + ); }); describe('CORS allows method GET and allows one origin', () => { @@ -172,12 +155,8 @@ describe('Preflight CORS request with existing bucket', () => { CORSConfiguration: { CORSRules: [ { - AllowedMethods: [ - 'GET', - ], - AllowedOrigins: [ - allowedOrigin, - ], + AllowedMethods: ['GET'], + AllowedOrigins: [allowedOrigin], }, ], }, @@ -194,52 +173,66 @@ describe('Preflight CORS request with existing bucket', () => { .catch(done); }); - it('should respond with 200 and access control headers to OPTIONS ' + - 'request from allowed origin and method "GET"', done => { - const headers = { - 'Origin': allowedOrigin, - 'Access-Control-Request-Method': 'GET', - }; - const headersResponse = { - 'access-control-allow-origin': allowedOrigin, - 'access-control-allow-methods': 'GET', - 'access-control-allow-credentials': 'true', - vary, - }; - methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, - headersResponse }, done); - }); - it('should respond AccessForbidden to OPTIONS request with allowed ' + - 'method but not from allowed origin', done => { - const headers = { - 'Origin': 'http://www.forbiddenwebsite.com', - 'Access-Control-Request-Method': 'GET', - }; - methodRequest({ method: 'OPTIONS', bucket, headers, - code: 'AccessForbidden', headersResponse: null }, done); - }); - it('should respond AccessForbidden to OPTIONS request from allowed ' + - 'origin and method but with not allowed Access-Control-Request-Headers', - done => { - const headers = { - 'Origin': allowedOrigin, - 'Access-Control-Request-Method': 'GET', - 'Access-Control-Request-Headers': 'Origin, Accept, ' + - 'Content-Type', - }; - methodRequest({ method: 'OPTIONS', bucket, headers, - code: 'AccessForbidden', headersResponse: null }, done); - }); - ['PUT', 'POST', 'DELETE'].forEach(method => { - it('should respond AccessForbidden to OPTIONS request from ' + - `allowed origin but not allowed method "${method}"`, done => { + it( + 'should respond with 200 and access control headers to OPTIONS ' + + 'request from allowed origin and method "GET"', + done => { const headers = { - 'Origin': allowedOrigin, - 'Access-Control-Request-Method': method, + Origin: allowedOrigin, + 'Access-Control-Request-Method': 'GET', }; - methodRequest({ method: 'OPTIONS', bucket, headers, - code: 'AccessForbidden', headersResponse: null }, done); - }); + const headersResponse = { + 'access-control-allow-origin': allowedOrigin, + 'access-control-allow-methods': 'GET', + 'access-control-allow-credentials': 'true', + vary, + }; + methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, headersResponse }, done); + }, + ); + it( + 'should respond AccessForbidden to OPTIONS request with allowed ' + 'method but not from allowed origin', + done => { + const headers = { + Origin: 'http://www.forbiddenwebsite.com', + 'Access-Control-Request-Method': 'GET', + }; + methodRequest( + { method: 'OPTIONS', bucket, headers, code: 'AccessForbidden', headersResponse: null }, + done, + ); + }, + ); + it( + 'should respond AccessForbidden to OPTIONS request from allowed ' + + 'origin and method but with not allowed Access-Control-Request-Headers', + done => { + const headers = { + Origin: allowedOrigin, + 'Access-Control-Request-Method': 'GET', + 'Access-Control-Request-Headers': 'Origin, Accept, ' + 'Content-Type', + }; + methodRequest( + { method: 'OPTIONS', bucket, headers, code: 'AccessForbidden', headersResponse: null }, + done, + ); + }, + ); + ['PUT', 'POST', 'DELETE'].forEach(method => { + it( + 'should respond AccessForbidden to OPTIONS request from ' + + `allowed origin but not allowed method "${method}"`, + done => { + const headers = { + Origin: allowedOrigin, + 'Access-Control-Request-Method': method, + }; + methodRequest( + { method: 'OPTIONS', bucket, headers, code: 'AccessForbidden', headersResponse: null }, + done, + ); + }, + ); }); }); @@ -255,8 +248,7 @@ describe('Preflight CORS request with existing bucket', () => { ], }, }; - describe(`CORS allows method "${allowedMethod}" and allows all origins`, - () => { + describe(`CORS allows method "${allowedMethod}" and allows all origins`, () => { beforeEach(done => { s3.send(new PutBucketCorsCommand(corsParams)) .then(() => done()) @@ -269,45 +261,56 @@ describe('Preflight CORS request with existing bucket', () => { .catch(done); }); - it('should respond with 200 and access control headers to ' + - `OPTIONS request from allowed origin and method "${allowedMethod}"`, - done => { - const headers = { - 'Origin': allowedOrigin, - 'Access-Control-Request-Method': allowedMethod, - }; - const headersResponse = { - 'access-control-allow-origin': '*', - 'access-control-allow-methods': allowedMethod, - vary, - }; - methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, - headersResponse }, done); - }); - it('should respond AccessForbidden to OPTIONS request from ' + - 'allowed origin and method but with not allowed Access-Control-' + - 'Request-Headers', done => { - const headers = { - 'Origin': allowedOrigin, - 'Access-Control-Request-Method': allowedMethod, - 'Access-Control-Request-Headers': 'Origin, Accept, ' + - 'Content-Type', - }; - methodRequest({ method: 'OPTIONS', bucket, headers, - code: 'AccessForbidden', headersResponse: null }, done); - }); - methods.filter(method => method !== allowedMethod) - .forEach(method => { - it('should respond AccessForbidden to OPTIONS request from ' + - `allowed origin but not allowed method "${method}"`, done => { + it( + 'should respond with 200 and access control headers to ' + + `OPTIONS request from allowed origin and method "${allowedMethod}"`, + done => { const headers = { - 'Origin': allowedOrigin, - 'Access-Control-Request-Method': method, + Origin: allowedOrigin, + 'Access-Control-Request-Method': allowedMethod, + }; + const headersResponse = { + 'access-control-allow-origin': '*', + 'access-control-allow-methods': allowedMethod, + vary, + }; + methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, headersResponse }, done); + }, + ); + it( + 'should respond AccessForbidden to OPTIONS request from ' + + 'allowed origin and method but with not allowed Access-Control-' + + 'Request-Headers', + done => { + const headers = { + Origin: allowedOrigin, + 'Access-Control-Request-Method': allowedMethod, + 'Access-Control-Request-Headers': 'Origin, Accept, ' + 'Content-Type', }; - methodRequest({ method: 'OPTIONS', bucket, headers, - code: 'AccessForbidden', headersResponse: null }, done); + methodRequest( + { method: 'OPTIONS', bucket, headers, code: 'AccessForbidden', headersResponse: null }, + done, + ); + }, + ); + methods + .filter(method => method !== allowedMethod) + .forEach(method => { + it( + 'should respond AccessForbidden to OPTIONS request from ' + + `allowed origin but not allowed method "${method}"`, + done => { + const headers = { + Origin: allowedOrigin, + 'Access-Control-Request-Method': method, + }; + methodRequest( + { method: 'OPTIONS', bucket, headers, code: 'AccessForbidden', headersResponse: null }, + done, + ); + }, + ); }); - }); }); }); @@ -339,54 +342,61 @@ describe('Preflight CORS request with existing bucket', () => { .catch(done); }); - [originWithoutWildcard, originReplaceWildcard] - .forEach(acceptableOrigin => { - it('should return 200 and CORS header to OPTIONS request ' + - `from allowed method and origin "${acceptableOrigin}"`, - done => { - const headers = { - 'Origin': acceptableOrigin, - 'Access-Control-Request-Method': 'GET', - }; - const headersResponse = { - 'access-control-allow-origin': acceptableOrigin, - 'access-control-allow-methods': 'GET', - 'access-control-allow-credentials': 'true', - vary, - }; - methodRequest({ method: 'OPTIONS', bucket, headers, - code: 200, headersResponse }, done); - }); + [originWithoutWildcard, originReplaceWildcard].forEach(acceptableOrigin => { + it( + 'should return 200 and CORS header to OPTIONS request ' + + `from allowed method and origin "${acceptableOrigin}"`, + done => { + const headers = { + Origin: acceptableOrigin, + 'Access-Control-Request-Method': 'GET', + }; + const headersResponse = { + 'access-control-allow-origin': acceptableOrigin, + 'access-control-allow-methods': 'GET', + 'access-control-allow-credentials': 'true', + vary, + }; + methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, headersResponse }, done); + }, + ); }); if (!origin.endsWith('*')) { - it('should respond AccessForbidden to OPTIONS request from ' + - `allowed method and origin "${originWithoutWildcard}test"`, - done => { - const headers = { - 'Origin': `${originWithoutWildcard}test`, - 'Access-Control-Request-Method': 'GET', - }; - methodRequest({ method: 'OPTIONS', bucket, headers, - code: 'AccessForbidden', headersResponse: null }, done); - }); + it( + 'should respond AccessForbidden to OPTIONS request from ' + + `allowed method and origin "${originWithoutWildcard}test"`, + done => { + const headers = { + Origin: `${originWithoutWildcard}test`, + 'Access-Control-Request-Method': 'GET', + }; + methodRequest( + { method: 'OPTIONS', bucket, headers, code: 'AccessForbidden', headersResponse: null }, + done, + ); + }, + ); } if (!origin.startsWith('*')) { - it('should respond AccessForbidden to OPTIONS request from ' + - `allowed method and origin "test${originWithoutWildcard}"`, - done => { - const headers = { - 'Origin': `test${originWithoutWildcard}`, - 'Access-Control-Request-Method': 'GET', - }; - methodRequest({ method: 'OPTIONS', bucket, headers, - code: 'AccessForbidden', headersResponse: null }, done); - }); + it( + 'should respond AccessForbidden to OPTIONS request from ' + + `allowed method and origin "test${originWithoutWildcard}"`, + done => { + const headers = { + Origin: `test${originWithoutWildcard}`, + 'Access-Control-Request-Method': 'GET', + }; + methodRequest( + { method: 'OPTIONS', bucket, headers, code: 'AccessForbidden', headersResponse: null }, + done, + ); + }, + ); } }); }); - describe('CORS response access-control-allow-origin header value', - () => { + describe('CORS response access-control-allow-origin header value', () => { const anotherOrigin = 'http://www.anotherorigin.com'; const originContainingWildcard = 'http://www.originwith*.com'; const corsParams = { @@ -394,21 +404,12 @@ describe('Preflight CORS request with existing bucket', () => { CORSConfiguration: { CORSRules: [ { - AllowedMethods: [ - 'GET', - ], - AllowedOrigins: [ - allowedOrigin, - originContainingWildcard, - ], + AllowedMethods: ['GET'], + AllowedOrigins: [allowedOrigin, originContainingWildcard], }, { - AllowedMethods: [ - 'GET', - ], - AllowedOrigins: [ - '*', - ], + AllowedMethods: ['GET'], + AllowedOrigins: ['*'], }, ], }, @@ -425,72 +426,71 @@ describe('Preflight CORS request with existing bucket', () => { .catch(done); }); - it('if OPTIONS request matches rule with multiple origins, response ' + - 'access-control-request-origin header value should be request Origin ' + - '(not list of AllowedOrigins)', done => { - const headers = { - 'Origin': allowedOrigin, - 'Access-Control-Request-Method': 'GET', - }; - const headersResponse = { - 'access-control-allow-origin': allowedOrigin, - 'access-control-allow-methods': 'GET', - 'access-control-allow-credentials': 'true', - vary, - }; - methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, - headersResponse }, done); - }); - it('if OPTIONS request matches rule with origin containing wildcard, ' + - 'response access-control-request-origin header value should be ' + - 'request Origin (not value containing wildcard)', done => { - const requestOrigin = originContainingWildcard.replace('*', 'test'); - const headers = { - 'Origin': requestOrigin, - 'Access-Control-Request-Method': 'GET', - }; - const headersResponse = { - 'access-control-allow-origin': requestOrigin, - 'access-control-allow-methods': 'GET', - 'access-control-allow-credentials': 'true', - vary, - }; - methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, - headersResponse }, done); - }); - it('if OPTIONS request matches rule that allows all origins, ' + - 'e.g. "*", response access-control-request-origin header should ' + - 'return "*"', done => { - const headers = { - 'Origin': anotherOrigin, - 'Access-Control-Request-Method': 'GET', - }; - const headersResponse = { - 'access-control-allow-origin': '*', - 'access-control-allow-methods': 'GET', - vary, - }; - methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, - headersResponse }, done); - }); + it( + 'if OPTIONS request matches rule with multiple origins, response ' + + 'access-control-request-origin header value should be request Origin ' + + '(not list of AllowedOrigins)', + done => { + const headers = { + Origin: allowedOrigin, + 'Access-Control-Request-Method': 'GET', + }; + const headersResponse = { + 'access-control-allow-origin': allowedOrigin, + 'access-control-allow-methods': 'GET', + 'access-control-allow-credentials': 'true', + vary, + }; + methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, headersResponse }, done); + }, + ); + it( + 'if OPTIONS request matches rule with origin containing wildcard, ' + + 'response access-control-request-origin header value should be ' + + 'request Origin (not value containing wildcard)', + done => { + const requestOrigin = originContainingWildcard.replace('*', 'test'); + const headers = { + Origin: requestOrigin, + 'Access-Control-Request-Method': 'GET', + }; + const headersResponse = { + 'access-control-allow-origin': requestOrigin, + 'access-control-allow-methods': 'GET', + 'access-control-allow-credentials': 'true', + vary, + }; + methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, headersResponse }, done); + }, + ); + it( + 'if OPTIONS request matches rule that allows all origins, ' + + 'e.g. "*", response access-control-request-origin header should ' + + 'return "*"', + done => { + const headers = { + Origin: anotherOrigin, + 'Access-Control-Request-Method': 'GET', + }; + const headersResponse = { + 'access-control-allow-origin': '*', + 'access-control-allow-methods': 'GET', + vary, + }; + methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, headersResponse }, done); + }, + ); }); - describe('CORS allows method GET, allows all origins and allows ' + - 'header Content-Type', () => { + describe('CORS allows method GET, allows all origins and allows ' + 'header Content-Type', () => { const corsParams = { Bucket: bucket, CORSConfiguration: { CORSRules: [ { - AllowedMethods: [ - 'GET', - ], - AllowedOrigins: [ - '*', - ], - AllowedHeaders: [ - 'content-type', - ], + AllowedMethods: ['GET'], + AllowedOrigins: ['*'], + AllowedHeaders: ['content-type'], }, ], }, @@ -507,80 +507,74 @@ describe('Preflight CORS request with existing bucket', () => { .catch(done); }); - it('should respond with 200 and access control headers to OPTIONS ' + - 'request from allowed origin and method, even without request ' + - 'Access-Control-Request-Headers header value', done => { - const headers = { - 'Origin': allowedOrigin, - 'Access-Control-Request-Method': 'GET', - }; - const headersResponse = { - 'access-control-allow-origin': '*', - 'access-control-allow-methods': 'GET', - vary, - }; - methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, - headersResponse }, done); - }); - it('should respond with 200 and access control headers to OPTIONS ' + - 'request from allowed origin and method with Access-Control-' + - 'Request-Headers \'Content-Type\'', done => { - const headers = { - 'Origin': allowedOrigin, - 'Access-Control-Request-Method': 'GET', - 'Access-Control-Request-Headers': 'content-type', - }; - const headersResponse = { - 'access-control-allow-origin': '*', - 'access-control-allow-methods': 'GET', - 'access-control-allow-headers': 'content-type', - vary, - }; - methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, - headersResponse }, done); - }); - it('should respond AccessForbidden to OPTIONS request from allowed ' + - 'origin and method but not allowed Access-Control-Request-Headers ' + - 'in addition to Content-Type', - done => { - const headers = { - 'Origin': allowedOrigin, - 'Access-Control-Request-Method': 'GET', - 'Access-Control-Request-Headers': 'Origin, Accept, ' + - 'content-type', - }; - methodRequest({ method: 'OPTIONS', bucket, headers, - code: 'AccessForbidden', headersResponse: null }, done); - }); + it( + 'should respond with 200 and access control headers to OPTIONS ' + + 'request from allowed origin and method, even without request ' + + 'Access-Control-Request-Headers header value', + done => { + const headers = { + Origin: allowedOrigin, + 'Access-Control-Request-Method': 'GET', + }; + const headersResponse = { + 'access-control-allow-origin': '*', + 'access-control-allow-methods': 'GET', + vary, + }; + methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, headersResponse }, done); + }, + ); + it( + 'should respond with 200 and access control headers to OPTIONS ' + + 'request from allowed origin and method with Access-Control-' + + "Request-Headers 'Content-Type'", + done => { + const headers = { + Origin: allowedOrigin, + 'Access-Control-Request-Method': 'GET', + 'Access-Control-Request-Headers': 'content-type', + }; + const headersResponse = { + 'access-control-allow-origin': '*', + 'access-control-allow-methods': 'GET', + 'access-control-allow-headers': 'content-type', + vary, + }; + methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, headersResponse }, done); + }, + ); + it( + 'should respond AccessForbidden to OPTIONS request from allowed ' + + 'origin and method but not allowed Access-Control-Request-Headers ' + + 'in addition to Content-Type', + done => { + const headers = { + Origin: allowedOrigin, + 'Access-Control-Request-Method': 'GET', + 'Access-Control-Request-Headers': 'Origin, Accept, ' + 'content-type', + }; + methodRequest( + { method: 'OPTIONS', bucket, headers, code: 'AccessForbidden', headersResponse: null }, + done, + ); + }, + ); }); - describe('CORS response Access-Control-Allow-Headers header value', - () => { + describe('CORS response Access-Control-Allow-Headers header value', () => { const corsParams = { Bucket: bucket, CORSConfiguration: { CORSRules: [ { - AllowedMethods: [ - 'GET', - ], - AllowedOrigins: [ - '*', - ], - AllowedHeaders: [ - 'Content-Type', 'amz-*', 'Expires', - ], + AllowedMethods: ['GET'], + AllowedOrigins: ['*'], + AllowedHeaders: ['Content-Type', 'amz-*', 'Expires'], }, { - AllowedMethods: [ - 'GET', - ], - AllowedOrigins: [ - '*', - ], - AllowedHeaders: [ - '*', - ], + AllowedMethods: ['GET'], + AllowedOrigins: ['*'], + AllowedHeaders: ['*'], }, ], }, @@ -597,82 +591,86 @@ describe('Preflight CORS request with existing bucket', () => { .catch(done); }); - it('should return request access-control-request-headers value, ' + - 'not list of AllowedHeaders from rule or corresponding AllowedHeader ' + - 'value containing wildcard', - done => { - const requestHeaderValue = 'amz-meta-header-test, content-type'; - const headers = { - 'Origin': allowedOrigin, - 'Access-Control-Request-Method': 'GET', - 'Access-Control-Request-Headers': requestHeaderValue, - }; - const headersResponse = { - 'access-control-allow-origin': '*', - 'access-control-allow-methods': 'GET', - 'access-control-allow-headers': requestHeaderValue, - vary, - }; - methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, - headersResponse }, done); - }); - it('should return lowercase version of request Access-Control-' + - 'Request-Method header value if it contains any upper-case values', - done => { - const requestHeaderValue = 'Content-Type'; - const headers = { - 'Origin': allowedOrigin, - 'Access-Control-Request-Method': 'GET', - 'Access-Control-Request-Headers': requestHeaderValue, - }; - const headersResponse = { - 'access-control-allow-origin': '*', - 'access-control-allow-methods': 'GET', - 'access-control-allow-headers': - requestHeaderValue.toLowerCase(), - vary, - }; - methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, - headersResponse }, done); - }); - it('should remove empty comma-separated values derived from request ' + - 'Access-Control-Request-Method header and separate values with ' + - 'spaces when responding with Access-Control-Allow-Headers header', - done => { - const requestHeaderValue = 'content-type,,expires'; - const expectedValue = 'content-type, expires'; - const headers = { - 'Origin': allowedOrigin, - 'Access-Control-Request-Method': 'GET', - 'Access-Control-Request-Headers': requestHeaderValue, - }; - const headersResponse = { - 'access-control-allow-origin': '*', - 'access-control-allow-methods': 'GET', - 'access-control-allow-headers': expectedValue, - vary, - }; - methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, - headersResponse }, done); - }); - it('should return request Access-Control-Request-Headers value ' + - 'even if rule allows all headers (e.g. "*"), unlike access-control-' + - 'allow-origin value', done => { - const requestHeaderValue = 'puppies'; - const headers = { - 'Origin': allowedOrigin, - 'Access-Control-Request-Method': 'GET', - 'Access-Control-Request-Headers': requestHeaderValue, - }; - const headersResponse = { - 'access-control-allow-origin': '*', - 'access-control-allow-methods': 'GET', - 'access-control-allow-headers': requestHeaderValue, - vary, - }; - methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, - headersResponse }, done); - }); + it( + 'should return request access-control-request-headers value, ' + + 'not list of AllowedHeaders from rule or corresponding AllowedHeader ' + + 'value containing wildcard', + done => { + const requestHeaderValue = 'amz-meta-header-test, content-type'; + const headers = { + Origin: allowedOrigin, + 'Access-Control-Request-Method': 'GET', + 'Access-Control-Request-Headers': requestHeaderValue, + }; + const headersResponse = { + 'access-control-allow-origin': '*', + 'access-control-allow-methods': 'GET', + 'access-control-allow-headers': requestHeaderValue, + vary, + }; + methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, headersResponse }, done); + }, + ); + it( + 'should return lowercase version of request Access-Control-' + + 'Request-Method header value if it contains any upper-case values', + done => { + const requestHeaderValue = 'Content-Type'; + const headers = { + Origin: allowedOrigin, + 'Access-Control-Request-Method': 'GET', + 'Access-Control-Request-Headers': requestHeaderValue, + }; + const headersResponse = { + 'access-control-allow-origin': '*', + 'access-control-allow-methods': 'GET', + 'access-control-allow-headers': requestHeaderValue.toLowerCase(), + vary, + }; + methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, headersResponse }, done); + }, + ); + it( + 'should remove empty comma-separated values derived from request ' + + 'Access-Control-Request-Method header and separate values with ' + + 'spaces when responding with Access-Control-Allow-Headers header', + done => { + const requestHeaderValue = 'content-type,,expires'; + const expectedValue = 'content-type, expires'; + const headers = { + Origin: allowedOrigin, + 'Access-Control-Request-Method': 'GET', + 'Access-Control-Request-Headers': requestHeaderValue, + }; + const headersResponse = { + 'access-control-allow-origin': '*', + 'access-control-allow-methods': 'GET', + 'access-control-allow-headers': expectedValue, + vary, + }; + methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, headersResponse }, done); + }, + ); + it( + 'should return request Access-Control-Request-Headers value ' + + 'even if rule allows all headers (e.g. "*"), unlike access-control-' + + 'allow-origin value', + done => { + const requestHeaderValue = 'puppies'; + const headers = { + Origin: allowedOrigin, + 'Access-Control-Request-Method': 'GET', + 'Access-Control-Request-Headers': requestHeaderValue, + }; + const headersResponse = { + 'access-control-allow-origin': '*', + 'access-control-allow-methods': 'GET', + 'access-control-allow-headers': requestHeaderValue, + vary, + }; + methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, headersResponse }, done); + }, + ); }); describe('CORS and OPTIONS request with object keys', () => { @@ -681,12 +679,8 @@ describe('Preflight CORS request with existing bucket', () => { CORSConfiguration: { CORSRules: [ { - AllowedMethods: [ - 'GET', - ], - AllowedOrigins: [ - allowedOrigin, - ], + AllowedMethods: ['GET'], + AllowedOrigins: [allowedOrigin], }, ], }, @@ -701,10 +695,14 @@ describe('Preflight CORS request with existing bucket', () => { afterEach(done => { s3.send(new DeleteBucketCorsCommand({ Bucket: bucket })) - .then(() => s3.send(new DeleteObjectCommand({ - Key: objectKey, - Bucket: bucket, - }))) + .then(() => + s3.send( + new DeleteObjectCommand({ + Key: objectKey, + Bucket: bucket, + }), + ), + ) .then(() => done()) .catch(err => { process.stdout.write(`err in afterEach ${err}`); @@ -712,38 +710,44 @@ describe('Preflight CORS request with existing bucket', () => { }); }); - it('should respond with 200 and access control headers to OPTIONS ' + - 'request from allowed origin, allowed method and existing object key', - done => { - const headers = { - 'Origin': allowedOrigin, - 'Access-Control-Request-Method': 'GET', - }; - const headersResponse = { - 'access-control-allow-origin': allowedOrigin, - 'access-control-allow-methods': 'GET', - 'access-control-allow-credentials': 'true', - vary, - }; - methodRequest({ method: 'OPTIONS', objectKey, bucket, headers, - code: 200, headersResponse }, done); - }); - it('should respond with 200 and access control headers to OPTIONS ' + - 'request from allowed origin, allowed method, even with non-existing ' + - 'object key', done => { - const headers = { - 'Origin': allowedOrigin, - 'Access-Control-Request-Method': 'GET', - }; - const headersResponse = { - 'access-control-allow-origin': allowedOrigin, - 'access-control-allow-methods': 'GET', - 'access-control-allow-credentials': 'true', - vary, - }; - methodRequest({ method: 'OPTIONS', bucket, objectKey: - 'anotherObjectKey', headers, code: 200, headersResponse }, done); - }); + it( + 'should respond with 200 and access control headers to OPTIONS ' + + 'request from allowed origin, allowed method and existing object key', + done => { + const headers = { + Origin: allowedOrigin, + 'Access-Control-Request-Method': 'GET', + }; + const headersResponse = { + 'access-control-allow-origin': allowedOrigin, + 'access-control-allow-methods': 'GET', + 'access-control-allow-credentials': 'true', + vary, + }; + methodRequest({ method: 'OPTIONS', objectKey, bucket, headers, code: 200, headersResponse }, done); + }, + ); + it( + 'should respond with 200 and access control headers to OPTIONS ' + + 'request from allowed origin, allowed method, even with non-existing ' + + 'object key', + done => { + const headers = { + Origin: allowedOrigin, + 'Access-Control-Request-Method': 'GET', + }; + const headersResponse = { + 'access-control-allow-origin': allowedOrigin, + 'access-control-allow-methods': 'GET', + 'access-control-allow-credentials': 'true', + vary, + }; + methodRequest( + { method: 'OPTIONS', bucket, objectKey: 'anotherObjectKey', headers, code: 200, headersResponse }, + done, + ); + }, + ); }); describe('CORS and OPTIONS request', () => { @@ -770,37 +774,41 @@ describe('Preflight CORS request with existing bucket', () => { .catch(done); }); - it('with fake auth credentials: should respond with 200 and access ' + - 'control headers even if request has fake auth credentials', done => { - const headers = { - 'Origin': allowedOrigin, - 'Access-Control-Request-Method': 'GET', - 'Authorization': 'AWS fakeKey:fakesignature', - }; - const headersResponse = { - 'access-control-allow-origin': '*', - 'access-control-allow-methods': 'GET', - vary, - }; - methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, - headersResponse }, done); - }); + it( + 'with fake auth credentials: should respond with 200 and access ' + + 'control headers even if request has fake auth credentials', + done => { + const headers = { + Origin: allowedOrigin, + 'Access-Control-Request-Method': 'GET', + Authorization: 'AWS fakeKey:fakesignature', + }; + const headersResponse = { + 'access-control-allow-origin': '*', + 'access-control-allow-methods': 'GET', + vary, + }; + methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, headersResponse }, done); + }, + ); - it('with cookies: should send identical response as to request ' + - 'without cookies (200 and access control headers)', done => { - const headers = { - 'Origin': allowedOrigin, - 'Access-Control-Request-Method': 'GET', - 'Cookie': 'testcookie=1', - }; - const headersResponse = { - 'access-control-allow-origin': '*', - 'access-control-allow-methods': 'GET', - vary, - }; - methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, - headersResponse }, done); - }); + it( + 'with cookies: should send identical response as to request ' + + 'without cookies (200 and access control headers)', + done => { + const headers = { + Origin: allowedOrigin, + 'Access-Control-Request-Method': 'GET', + Cookie: 'testcookie=1', + }; + const headersResponse = { + 'access-control-allow-origin': '*', + 'access-control-allow-methods': 'GET', + vary, + }; + methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, headersResponse }, done); + }, + ); }); describe('CORS exposes headers', () => { @@ -809,17 +817,9 @@ describe('Preflight CORS request with existing bucket', () => { CORSConfiguration: { CORSRules: [ { - AllowedMethods: [ - 'GET', - ], - AllowedOrigins: [ - '*', - ], - ExposeHeaders: [ - 'x-amz-server-side-encryption', - 'x-amz-request-id', - 'x-amz-id-2', - ], + AllowedMethods: ['GET'], + AllowedOrigins: ['*'], + ExposeHeaders: ['x-amz-server-side-encryption', 'x-amz-request-id', 'x-amz-id-2'], }, ], }, @@ -836,23 +836,23 @@ describe('Preflight CORS request with existing bucket', () => { .catch(done); }); - it('if OPTIONS request matches CORS rule with ExposeHeader\'s, ' + - 'response should include Access-Control-Expose-Headers header', - done => { - const headers = { - 'Origin': allowedOrigin, - 'Access-Control-Request-Method': 'GET', - }; - const headersResponse = { - 'access-control-allow-origin': '*', - 'access-control-allow-methods': 'GET', - 'access-control-expose-headers': - 'x-amz-server-side-encryption, x-amz-request-id, x-amz-id-2', - vary, - }; - methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, - headersResponse }, done); - }); + it( + "if OPTIONS request matches CORS rule with ExposeHeader's, " + + 'response should include Access-Control-Expose-Headers header', + done => { + const headers = { + Origin: allowedOrigin, + 'Access-Control-Request-Method': 'GET', + }; + const headersResponse = { + 'access-control-allow-origin': '*', + 'access-control-allow-methods': 'GET', + 'access-control-expose-headers': 'x-amz-server-side-encryption, x-amz-request-id, x-amz-id-2', + vary, + }; + methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, headersResponse }, done); + }, + ); }); describe('CORS max age seconds', () => { @@ -861,12 +861,8 @@ describe('Preflight CORS request with existing bucket', () => { CORSConfiguration: { CORSRules: [ { - AllowedMethods: [ - 'GET', - ], - AllowedOrigins: [ - '*', - ], + AllowedMethods: ['GET'], + AllowedOrigins: ['*'], MaxAgeSeconds: 86400, }, ], @@ -884,20 +880,22 @@ describe('Preflight CORS request with existing bucket', () => { .catch(done); }); - it('if OPTIONS request matches CORS rule with max age seconds, ' + - 'response should include Access-Control-Max-Age header', done => { - const headers = { - 'Origin': allowedOrigin, - 'Access-Control-Request-Method': 'GET', - }; - const headersResponse = { - 'access-control-allow-origin': '*', - 'access-control-allow-methods': 'GET', - 'access-control-max-age': '86400', - vary, - }; - methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, - headersResponse }, done); - }); + it( + 'if OPTIONS request matches CORS rule with max age seconds, ' + + 'response should include Access-Control-Max-Age header', + done => { + const headers = { + Origin: allowedOrigin, + 'Access-Control-Request-Method': 'GET', + }; + const headersResponse = { + 'access-control-allow-origin': '*', + 'access-control-allow-methods': 'GET', + 'access-control-max-age': '86400', + vary, + }; + methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, headersResponse }, done); + }, + ); }); }); diff --git a/tests/functional/aws-node-sdk/test/object/deleteMpu.js b/tests/functional/aws-node-sdk/test/object/deleteMpu.js index 60fdad6f13..d6a054046a 100644 --- a/tests/functional/aws-node-sdk/test/object/deleteMpu.js +++ b/tests/functional/aws-node-sdk/test/object/deleteMpu.js @@ -18,8 +18,8 @@ const westLocation = 'scality-us-west-1'; const eastLocation = 'us-east-1'; const confLocations = [ - { name: 'us-west-1', statusCode: 204, location: westLocation, describe }, - { name: 'us-east-1', statusCode: 404, location: eastLocation, describe }, + { name: 'us-west-1', statusCode: 204, location: westLocation, describe }, + { name: 'us-east-1', statusCode: 404, location: eastLocation, describe }, ]; describe('DELETE multipart', () => { @@ -34,19 +34,17 @@ describe('DELETE multipart', () => { UploadId: uploadId, }); - s3Client.send(command) + s3Client + .send(command) .then(response => { - const statusCode = - response?.$metadata?.httpStatusCode; - assert.strictEqual(statusCode, statusCodeExpected, - `Found unexpected statusCode ${statusCode}`); + const statusCode = response?.$metadata?.httpStatusCode; + assert.strictEqual(statusCode, statusCodeExpected, `Found unexpected statusCode ${statusCode}`); return callback(); }) .catch(err => { const statusCode = err?.$metadata?.httpStatusCode; if (statusCode) { - assert.strictEqual(statusCode, statusCodeExpected, - `Found unexpected statusCode ${statusCode}`); + assert.strictEqual(statusCode, statusCodeExpected, `Found unexpected statusCode ${statusCode}`); } if (statusCodeExpected === 204) { return callback(err); @@ -55,8 +53,7 @@ describe('DELETE multipart', () => { }); } - it('on bucket that does not exist: should return NoSuchBucket', - done => { + it('on bucket that does not exist: should return NoSuchBucket', done => { const uploadId = 'nonexistinguploadid'; const command = new AbortMultipartUploadCommand({ Bucket: bucket, @@ -64,22 +61,20 @@ describe('DELETE multipart', () => { UploadId: uploadId, }); - s3Client.send(command) + s3Client + .send(command) .then(() => { done(new Error('Expected NoSuchBucket but request succeeded')); }) .catch(err => { - assert.notEqual(err, null, - 'Expected NoSuchBucket but found no err'); + assert.notEqual(err, null, 'Expected NoSuchBucket but found no err'); assert.strictEqual(err.name, 'NoSuchBucket'); done(); }); }); confLocations.forEach(confLocation => { - confLocation.describe('on existing bucket with ' + - `${confLocation.name}`, - () => { + confLocation.describe('on existing bucket with ' + `${confLocation.name}`, () => { beforeEach(async () => { const command = new CreateBucketCommand({ Bucket: bucket, @@ -91,38 +86,38 @@ describe('DELETE multipart', () => { }); afterEach(async () => { - process.stdout.write('Emptying bucket\n'); - await bucketUtil.empty(bucket); - process.stdout.write('Deleting bucket\n'); - await bucketUtil.deleteOne(bucket); + process.stdout.write('Emptying bucket\n'); + await bucketUtil.empty(bucket); + process.stdout.write('Deleting bucket\n'); + await bucketUtil.deleteOne(bucket); }); - itSkipIfAWS(`should return ${confLocation.statusCode} if ` + - 'mpu does not exist with uploadId', - done => { - const uploadId = 'nonexistinguploadid'; - _assertStatusCode(uploadId, confLocation.statusCode, done); - }); + itSkipIfAWS( + `should return ${confLocation.statusCode} if ` + 'mpu does not exist with uploadId', + done => { + const uploadId = 'nonexistinguploadid'; + _assertStatusCode(uploadId, confLocation.statusCode, done); + }, + ); - describe('if mpu exists with uploadId + at least one part', - () => { + describe('if mpu exists with uploadId + at least one part', () => { let uploadId; beforeEach(async () => { - const createCommand = new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: key, - }); - const createResponse = await s3Client.send(createCommand); - uploadId = createResponse.UploadId; - const uploadCommand = new UploadPartCommand({ - Bucket: bucket, - Key: key, - PartNumber: 1, - UploadId: uploadId, - Body: Buffer.from('test data'), - }); - await s3Client.send(uploadCommand); + const createCommand = new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + }); + const createResponse = await s3Client.send(createCommand); + uploadId = createResponse.UploadId; + const uploadCommand = new UploadPartCommand({ + Bucket: bucket, + Key: key, + PartNumber: 1, + UploadId: uploadId, + Body: Buffer.from('test data'), + }); + await s3Client.send(uploadCommand); }); it('should return 204 for abortMultipartUpload', done => { diff --git a/tests/functional/aws-node-sdk/test/object/deleteObjTagging.js b/tests/functional/aws-node-sdk/test/object/deleteObjTagging.js index d8839a5760..da40f392a1 100644 --- a/tests/functional/aws-node-sdk/test/object/deleteObjTagging.js +++ b/tests/functional/aws-node-sdk/test/object/deleteObjTagging.js @@ -16,16 +16,18 @@ const bucketName = 'testdeletetaggingbucket'; const objectName = 'testtaggingobject'; const objectNameAcl = 'testtaggingobjectacl'; -const taggingConfig = { TagSet: [ - { - Key: 'key1', - Value: 'value1', - }, - { - Key: 'key2', - Value: 'value2', - }, -] }; +const taggingConfig = { + TagSet: [ + { + Key: 'key1', + Value: 'value1', + }, + { + Key: 'key2', + Value: 'value2', + }, + ], +}; function _checkError(err, code, statusCode) { assert(err, 'Expected error but found none'); @@ -53,120 +55,155 @@ describe('DELETE object taggings', () => { }); it('should delete tag set', async () => { - await s3.send(new PutObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - Tagging: taggingConfig, - })); - await s3.send(new DeleteObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName - })); - const dataGet = await s3.send(new GetObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - })); + await s3.send( + new PutObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + Tagging: taggingConfig, + }), + ); + await s3.send( + new DeleteObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); + const dataGet = await s3.send( + new GetObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); assert.strictEqual(dataGet.TagSet.length, 0); }); it('should delete a non-existing tag set', async () => { - await s3.send(new DeleteObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName - })); - const dataGet = await s3.send(new GetObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName - })); + await s3.send( + new DeleteObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); + const dataGet = await s3.send( + new GetObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); assert.strictEqual(dataGet.TagSet.length, 0); }); - it('should return NoSuchKey deleting tag set to a non-existing object', - async () => { + it('should return NoSuchKey deleting tag set to a non-existing object', async () => { try { - await s3.send(new DeleteObjectTaggingCommand({ - Bucket: bucketName, - Key: 'nonexisting', - })); + await s3.send( + new DeleteObjectTaggingCommand({ + Bucket: bucketName, + Key: 'nonexisting', + }), + ); assert.fail('Expected NoSuchKey error'); } catch (err) { _checkError(err, 'NoSuchKey', 404); } }); - it('should return 403 AccessDenied deleting tag set with another ' + - 'account', async () => { + it('should return 403 AccessDenied deleting tag set with another ' + 'account', async () => { try { - await otherAccountS3.send(new DeleteObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName - })); + await otherAccountS3.send( + new DeleteObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); assert.fail('Expected AccessDenied error'); } catch (err) { _checkError(err, 'AccessDenied', 403); } }); - it('should return 403 AccessDenied deleting tag set with a different ' + - 'account to an object with ACL "public-read-write"', + it( + 'should return 403 AccessDenied deleting tag set with a different ' + + 'account to an object with ACL "public-read-write"', async () => { - await s3.send(new PutObjectAclCommand({ - Bucket: bucketName, - Key: objectName, - ACL: 'public-read-write' - })); - - try { - await otherAccountS3.send(new DeleteObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName - })); - assert.fail('Expected AccessDenied error'); - } catch (err) { - _checkError(err, 'AccessDenied', 403); - } - }); + await s3.send( + new PutObjectAclCommand({ + Bucket: bucketName, + Key: objectName, + ACL: 'public-read-write', + }), + ); + + try { + await otherAccountS3.send( + new DeleteObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); + assert.fail('Expected AccessDenied error'); + } catch (err) { + _checkError(err, 'AccessDenied', 403); + } + }, + ); - it('should return 403 AccessDenied deleting tag set to an object '+ - ' in a bucket created with a different account', + it( + 'should return 403 AccessDenied deleting tag set to an object ' + + ' in a bucket created with a different account', async () => { - await s3.send(new PutBucketAclCommand({ - Bucket: bucketName, - ACL: 'public-read-write' - })); - - await otherAccountS3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectNameAcl - })); - - try { - await otherAccountS3.send(new DeleteObjectTaggingCommand({ - Bucket: bucketName, - Key: objectNameAcl - })); - assert.fail('Expected AccessDenied error'); - } catch (err) { - _checkError(err, 'AccessDenied', 403); - } - }); + await s3.send( + new PutBucketAclCommand({ + Bucket: bucketName, + ACL: 'public-read-write', + }), + ); - it('should delete tag set to an object in a bucket created with '+ - 'same account even though object put by other account', async () => { - await s3.send(new PutBucketAclCommand({ - Bucket: bucketName, - ACL: 'public-read-write' - })); - - await otherAccountS3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectNameAcl - })); - - await s3.send(new DeleteObjectTaggingCommand({ - Bucket: bucketName, - Key: objectNameAcl - })); - }); + await otherAccountS3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectNameAcl, + }), + ); + + try { + await otherAccountS3.send( + new DeleteObjectTaggingCommand({ + Bucket: bucketName, + Key: objectNameAcl, + }), + ); + assert.fail('Expected AccessDenied error'); + } catch (err) { + _checkError(err, 'AccessDenied', 403); + } + }, + ); + + it( + 'should delete tag set to an object in a bucket created with ' + + 'same account even though object put by other account', + async () => { + await s3.send( + new PutBucketAclCommand({ + Bucket: bucketName, + ACL: 'public-read-write', + }), + ); + + await otherAccountS3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectNameAcl, + }), + ); + + await s3.send( + new DeleteObjectTaggingCommand({ + Bucket: bucketName, + Key: objectNameAcl, + }), + ); + }, + ); }); }); diff --git a/tests/functional/aws-node-sdk/test/object/deleteObject.js b/tests/functional/aws-node-sdk/test/object/deleteObject.js index ee14d27d06..ccfd453af5 100644 --- a/tests/functional/aws-node-sdk/test/object/deleteObject.js +++ b/tests/functional/aws-node-sdk/test/object/deleteObject.js @@ -1,7 +1,7 @@ const assert = require('assert'); const moment = require('moment'); const { - CreateBucketCommand, + CreateBucketCommand, CreateMultipartUploadCommand, UploadPartCommand, CompleteMultipartUploadCommand, @@ -10,7 +10,7 @@ const { PutObjectRetentionCommand, PutObjectLegalHoldCommand, PutObjectLockConfigurationCommand, - HeadObjectCommand + HeadObjectCommand, } = require('@aws-sdk/client-s3'); const withV4 = require('../support/withV4'); const BucketUtility = require('../../lib/utility/bucket-util'); @@ -18,7 +18,6 @@ const changeObjectLock = require('../../../../utilities/objectLock-util'); const objectName = 'key'; const objectNameTwo = 'secondkey'; - describe('DELETE object', () => { withV4(sigCfg => { let uploadId; @@ -32,13 +31,15 @@ describe('DELETE object', () => { try { process.stdout.write('creating bucket\n'); await s3.send(new CreateBucketCommand({ Bucket: bucketName })); - + process.stdout.write('initiating multipart upload\n'); - const createRes = await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucketName, - Key: objectName, - })); - + const createRes = await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); + process.stdout.write('uploading parts\n'); uploadId = createRes.UploadId; // Concurrent uploads help trigger flakiness with "TimeoutError: @@ -47,37 +48,43 @@ describe('DELETE object', () => { const uploadWithRetry = (params, attempt = 0) => s3.send(new UploadPartCommand(params)).catch(err => { if (attempt < 3) { - process.stdout.write(`Retrying UploadPart ${params.PartNumber} ` - + `(attempt ${attempt + 1}/3): ${err}\n`); + process.stdout.write( + `Retrying UploadPart ${params.PartNumber} ` + + `(attempt ${attempt + 1}/3): ${err}\n`, + ); return uploadWithRetry(params, attempt + 1); } throw err; }); const uploads = []; for (let i = 1; i <= 3; i++) { - uploads.push(uploadWithRetry({ - Bucket: bucketName, - Key: objectName, - PartNumber: i, - Body: testfile, - UploadId: uploadId, - })); + uploads.push( + uploadWithRetry({ + Bucket: bucketName, + Key: objectName, + PartNumber: i, + Body: testfile, + UploadId: uploadId, + }), + ); } const uploadResults = await Promise.all(uploads); - + process.stdout.write('about to complete multipart upload\n'); - await s3.send(new CompleteMultipartUploadCommand({ - Bucket: bucketName, - Key: objectName, - UploadId: uploadId, - MultipartUpload: { - Parts: [ - { ETag: uploadResults[0].ETag, PartNumber: 1 }, - { ETag: uploadResults[1].ETag, PartNumber: 2 }, - { ETag: uploadResults[2].ETag, PartNumber: 3 }, - ], - }, - })); + await s3.send( + new CompleteMultipartUploadCommand({ + Bucket: bucketName, + Key: objectName, + UploadId: uploadId, + MultipartUpload: { + Parts: [ + { ETag: uploadResults[0].ETag, PartNumber: 1 }, + { ETag: uploadResults[1].ETag, PartNumber: 2 }, + { ETag: uploadResults[2].ETag, PartNumber: 3 }, + ], + }, + }), + ); } catch (err) { process.stdout.write(`Error in before: ${err}\n`); throw err; @@ -112,47 +119,57 @@ describe('DELETE object', () => { let versionIdOne; let versionIdTwo; const retainDate = moment().add(10, 'days').toDate(); - + before(async () => { try { process.stdout.write('creating bucket\n'); - await s3.send(new CreateBucketCommand({ - Bucket: bucketName, - ObjectLockEnabledForBucket: true, - })); - + await s3.send( + new CreateBucketCommand({ + Bucket: bucketName, + ObjectLockEnabledForBucket: true, + }), + ); + process.stdout.write('putting object\n'); - const res1 = await s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName, - })); + const res1 = await s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); versionIdOne = res1.VersionId; - + process.stdout.write('putting object retention\n'); - await s3.send(new PutObjectRetentionCommand({ - Bucket: bucketName, - Key: objectName, - Retention: { - Mode: 'GOVERNANCE', - RetainUntilDate: retainDate, - }, - })); - + await s3.send( + new PutObjectRetentionCommand({ + Bucket: bucketName, + Key: objectName, + Retention: { + Mode: 'GOVERNANCE', + RetainUntilDate: retainDate, + }, + }), + ); + process.stdout.write('putting object\n'); - const res2 = await s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectNameTwo, - })); + const res2 = await s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectNameTwo, + }), + ); versionIdTwo = res2.VersionId; - + process.stdout.write('putting object legal hold\n'); - await s3.send(new PutObjectLegalHoldCommand({ - Bucket: bucketName, - Key: objectNameTwo, - LegalHold: { - Status: 'ON', - }, - })); + await s3.send( + new PutObjectLegalHoldCommand({ + Bucket: bucketName, + Key: objectNameTwo, + LegalHold: { + Status: 'ON', + }, + }), + ); } catch (err) { process.stdout.write(`Error in before: ${err}\n`); throw err; @@ -165,10 +182,12 @@ describe('DELETE object', () => { }); it('should put delete marker if no version id specified', done => { - s3.send(new DeleteObjectCommand({ - Bucket: bucketName, - Key: objectName, - })) + s3.send( + new DeleteObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ) .then(() => { done(); }) @@ -179,11 +198,13 @@ describe('DELETE object', () => { }); it('should not delete object version locked with object retention', done => { - s3.send(new DeleteObjectCommand({ - Bucket: bucketName, - Key: objectName, - VersionId: versionIdOne, - })) + s3.send( + new DeleteObjectCommand({ + Bucket: bucketName, + Key: objectName, + VersionId: versionIdOne, + }), + ) .then(() => { assert.fail('Should have failed'); }) @@ -194,12 +215,14 @@ describe('DELETE object', () => { }); it('should delete locked object version with GOVERNANCE retention mode and correct header', done => { - s3.send(new DeleteObjectCommand({ - Bucket: bucketName, - Key: objectName, - VersionId: versionIdOne, - BypassGovernanceRetention: true, - })) + s3.send( + new DeleteObjectCommand({ + Bucket: bucketName, + Key: objectName, + VersionId: versionIdOne, + BypassGovernanceRetention: true, + }), + ) .then(() => { done(); }) @@ -210,19 +233,25 @@ describe('DELETE object', () => { }); it('should not delete object locked with legal hold', done => { - s3.send(new DeleteObjectCommand({ - Bucket: bucketName, - Key: objectNameTwo, - VersionId: versionIdTwo, - })) - .catch(err => { + s3.send( + new DeleteObjectCommand({ + Bucket: bucketName, + Key: objectNameTwo, + VersionId: versionIdTwo, + }), + ).catch(err => { assert.strictEqual(err.name, 'AccessDenied'); changeObjectLock( - [{ - bucket: bucketName, - key: objectNameTwo, - versionId: versionIdTwo, - }], '', done); + [ + { + bucket: bucketName, + key: objectNameTwo, + versionId: versionIdTwo, + }, + ], + '', + done, + ); }); }); }); @@ -231,44 +260,52 @@ describe('DELETE object', () => { const bucketName = 'testdeletelocklegalholdbucket'; const objectName = 'key'; let versionId; - + before(async () => { try { process.stdout.write('creating bucket\n'); - await s3.send(new CreateBucketCommand({ - Bucket: bucketName, - ObjectLockEnabledForBucket: true, - })); - + await s3.send( + new CreateBucketCommand({ + Bucket: bucketName, + ObjectLockEnabledForBucket: true, + }), + ); + process.stdout.write('putting object lock configuration\n'); - await s3.send(new PutObjectLockConfigurationCommand({ - Bucket: bucketName, - ObjectLockConfiguration: { - ObjectLockEnabled: 'Enabled', - Rule: { - DefaultRetention: { - Mode: 'GOVERNANCE', - Days: 1, + await s3.send( + new PutObjectLockConfigurationCommand({ + Bucket: bucketName, + ObjectLockConfiguration: { + ObjectLockEnabled: 'Enabled', + Rule: { + DefaultRetention: { + Mode: 'GOVERNANCE', + Days: 1, + }, }, }, - }, - })); - + }), + ); + process.stdout.write('putting object\n'); - const res = await s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName, - })); + const res = await s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); versionId = res.VersionId; - + process.stdout.write('putting object legal hold\n'); - await s3.send(new PutObjectLegalHoldCommand({ - Bucket: bucketName, - Key: objectName, - LegalHold: { - Status: 'ON', - }, - })); + await s3.send( + new PutObjectLegalHoldCommand({ + Bucket: bucketName, + Key: objectName, + LegalHold: { + Status: 'ON', + }, + }), + ); } catch (err) { process.stdout.write(`Error in before: ${err}\n`); throw err; @@ -287,24 +324,33 @@ describe('DELETE object', () => { } }); - it('should not delete locked object version with GOVERNANCE ' + - 'retention mode and bypass header when object is legal-hold enabled', done => { - s3.send(new DeleteObjectCommand({ - Bucket: bucketName, - Key: objectName, - VersionId: versionId, - BypassGovernanceRetention: true, - })) - .catch(err => { - assert.strictEqual(err.name, 'AccessDenied'); - changeObjectLock( - [{ - bucket: bucketName, - key: objectName, - versionId, - }], '', done); - }); - }); + it( + 'should not delete locked object version with GOVERNANCE ' + + 'retention mode and bypass header when object is legal-hold enabled', + done => { + s3.send( + new DeleteObjectCommand({ + Bucket: bucketName, + Key: objectName, + VersionId: versionId, + BypassGovernanceRetention: true, + }), + ).catch(err => { + assert.strictEqual(err.name, 'AccessDenied'); + changeObjectLock( + [ + { + bucket: bucketName, + key: objectName, + versionId, + }, + ], + '', + done, + ); + }); + }, + ); }); describe('with conditional headers (unofficial, for backbeat)', () => { @@ -319,15 +365,19 @@ describe('DELETE object', () => { beforeEach(async () => { // Re-create the object for each test since some tests will delete it - await s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: testObjectKey, - Body: testObjectBody, - })); - const head = await s3.send(new HeadObjectCommand({ - Bucket: bucketName, - Key: testObjectKey, - })); + await s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: testObjectKey, + Body: testObjectBody, + }), + ); + const head = await s3.send( + new HeadObjectCommand({ + Bucket: bucketName, + Key: testObjectKey, + }), + ); objectLastModified = head.LastModified; }); @@ -340,7 +390,7 @@ describe('DELETE object', () => { const command = new DeleteObjectCommand(params); // Create a unique middleware name to avoid conflicts const middlewareName = `headersAdder_${Date.now()}_${Math.random()}`; - + // Middleware to add custom headers const middleware = next => async args => { for (const [key, value] of Object.entries(headers)) { @@ -350,15 +400,15 @@ describe('DELETE object', () => { } return next(args); }; - + const middlewareConfig = { step: 'build', name: middlewareName, }; - + // Add middleware s3.middlewareStack.add(middleware, middlewareConfig); - + s3.send(command) .then(data => { s3.middlewareStack.remove(middlewareName); @@ -374,41 +424,53 @@ describe('DELETE object', () => { it('should delete when condition is true (date after object modification)', done => { const futureDate = new Date(objectLastModified.getTime() + 60_000); // 1 minute later - deleteObjectConditional(s3, { - Bucket: bucketName, - Key: testObjectKey, - }, { - 'If-Unmodified-Since': futureDate.toUTCString(), - }, (err, data) => { - assert.ifError(err); - assert.deepStrictEqual(data.$metadata.httpStatusCode, 204); - s3.send(new HeadObjectCommand({ + deleteObjectConditional( + s3, + { Bucket: bucketName, Key: testObjectKey, - })) - .then(() => { - assert.fail('Object should not exist'); - }) - .catch(err => { - assert.strictEqual(err.name, 'NotFound'); - done(); - }); - }); + }, + { + 'If-Unmodified-Since': futureDate.toUTCString(), + }, + (err, data) => { + assert.ifError(err); + assert.deepStrictEqual(data.$metadata.httpStatusCode, 204); + s3.send( + new HeadObjectCommand({ + Bucket: bucketName, + Key: testObjectKey, + }), + ) + .then(() => { + assert.fail('Object should not exist'); + }) + .catch(err => { + assert.strictEqual(err.name, 'NotFound'); + done(); + }); + }, + ); }); it('should fail (412) to delete when condition is false (date before object modification)', done => { const pastDate = new Date(objectLastModified.getTime() - 60_000); // 1 minute earlier - deleteObjectConditional(s3, { - Bucket: bucketName, - Key: testObjectKey, - }, { - 'If-Unmodified-Since': pastDate.toUTCString(), - }, err => { - assert.strictEqual(err.name, 'PreconditionFailed'); - assert.strictEqual(err.$metadata.httpStatusCode, 412); - done(); - }); + deleteObjectConditional( + s3, + { + Bucket: bucketName, + Key: testObjectKey, + }, + { + 'If-Unmodified-Since': pastDate.toUTCString(), + }, + err => { + assert.strictEqual(err.name, 'PreconditionFailed'); + assert.strictEqual(err.$metadata.httpStatusCode, 412); + done(); + }, + ); }); }); @@ -416,36 +478,47 @@ describe('DELETE object', () => { it('should delete when condition is true (date before object modification)', done => { const pastDate = new Date(objectLastModified.getTime() - 60_000); // 1 minute earlier - deleteObjectConditional(s3, { - Bucket: bucketName, - Key: testObjectKey, - }, { - 'If-Modified-Since': pastDate.toUTCString() - }, (err, data) => { - assert.deepStrictEqual(data.$metadata.httpStatusCode, 204); - s3.send(new HeadObjectCommand({ + deleteObjectConditional( + s3, + { Bucket: bucketName, Key: testObjectKey, - })) - .catch(err => { - assert.strictEqual(err.name, 'NotFound'); - done(); - }); - }); + }, + { + 'If-Modified-Since': pastDate.toUTCString(), + }, + (err, data) => { + assert.deepStrictEqual(data.$metadata.httpStatusCode, 204); + s3.send( + new HeadObjectCommand({ + Bucket: bucketName, + Key: testObjectKey, + }), + ).catch(err => { + assert.strictEqual(err.name, 'NotFound'); + done(); + }); + }, + ); }); it('should fail (304) to delete when condition is false (date after object modification)', done => { const futureDate = new Date(objectLastModified.getTime() + 60_000); // 1 minute later - deleteObjectConditional(s3, { - Bucket: bucketName, - Key: testObjectKey, - }, { - 'If-Modified-Since': futureDate.toUTCString(), - }, err => { - assert.strictEqual(err.$metadata.httpStatusCode, 304); - done(); - }); + deleteObjectConditional( + s3, + { + Bucket: bucketName, + Key: testObjectKey, + }, + { + 'If-Modified-Since': futureDate.toUTCString(), + }, + err => { + assert.strictEqual(err.$metadata.httpStatusCode, 304); + done(); + }, + ); }); }); @@ -454,24 +527,30 @@ describe('DELETE object', () => { const pastDate = new Date(objectLastModified.getTime() - 60_000); // 1 minute earlier const futureDate = new Date(objectLastModified.getTime() + 60_000); // 1 minute later - deleteObjectConditional(s3, { - Bucket: bucketName, - Key: testObjectKey, - }, { - 'If-Modified-Since': pastDate.toUTCString(), - 'If-Unmodified-Since': futureDate.toUTCString(), - }, (err, data) => { - assert.deepStrictEqual(data.$metadata.httpStatusCode, 204); - s3.send(new HeadObjectCommand({ + deleteObjectConditional( + s3, + { Bucket: bucketName, Key: testObjectKey, - })) - .catch(err => { - assert.strictEqual(err.name, 'NotFound'); - assert.strictEqual(err.$metadata.httpStatusCode, 404); - done(); - }); - }); + }, + { + 'If-Modified-Since': pastDate.toUTCString(), + 'If-Unmodified-Since': futureDate.toUTCString(), + }, + (err, data) => { + assert.deepStrictEqual(data.$metadata.httpStatusCode, 204); + s3.send( + new HeadObjectCommand({ + Bucket: bucketName, + Key: testObjectKey, + }), + ).catch(err => { + assert.strictEqual(err.name, 'NotFound'); + assert.strictEqual(err.$metadata.httpStatusCode, 404); + done(); + }); + }, + ); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/object/encryptionHeaders.js b/tests/functional/aws-node-sdk/test/object/encryptionHeaders.js index d4ec55375a..3a765a224e 100644 --- a/tests/functional/aws-node-sdk/test/object/encryptionHeaders.js +++ b/tests/functional/aws-node-sdk/test/object/encryptionHeaders.js @@ -1,7 +1,7 @@ const assert = require('assert'); const async = require('async'); const uuid = require('uuid'); -const { +const { CreateBucketCommand, HeadObjectCommand, PutObjectCommand, @@ -37,16 +37,20 @@ const testCases = [ }, ]; -function s3NoOp(_, cb) { cb(); } +function s3NoOp(_, cb) { + cb(); +} function getSSEConfig(s3, Bucket, Key, cb) { const command = new HeadObjectCommand({ Bucket, Key }); s3.send(command) .then(resp => { - const sseConfig = JSON.parse(JSON.stringify({ - algo: resp.ServerSideEncryption, - masterKeyId: resp.SSEKMSKeyId - })); + const sseConfig = JSON.parse( + JSON.stringify({ + algo: resp.ServerSideEncryption, + masterKeyId: resp.SSEKMSKeyId, + }), + ); cb(null, sseConfig); }) .catch(cb); @@ -58,15 +62,15 @@ function putEncryptedObject(s3, Bucket, Key, sseConfig, kmsKeyId, cb) { Key, Body: 'somedata', }; - + if (sseConfig.algo) { params.ServerSideEncryption = sseConfig.algo; } - + if (sseConfig.masterKeyId) { params.SSEKMSKeyId = kmsKeyId; } - + const command = new PutObjectCommand(params); s3.send(command) .then(response => cb(null, response)) @@ -80,9 +84,7 @@ function createExpected(sseConfig, kmsKeyId) { } if (sseConfig.masterKeyId) { - expected.masterKeyId = config.kmsHideScalityArn - ? getKeyIdFromArn(kmsKeyId) - : kmsKeyId; + expected.masterKeyId = config.kmsHideScalityArn ? getKeyIdFromArn(kmsKeyId) : kmsKeyId; } return expected; } @@ -99,8 +101,7 @@ function hydrateSSEConfig({ algo: SSEAlgorithm, masterKeyId: KMSMasterKeyID }) { }, }, ], - } - ) + }), ); } @@ -143,15 +144,12 @@ describe('per object encryption headers', () => { let kmsKeyId; before(done => { - const bucket = new BucketInfo('enc-bucket-test', 'OwnerId', - 'OwnerDisplayName', new Date().toJSON()); - kms.createBucketKey(bucket, log, - (err, { masterKeyArn: keyId }) => { - assert.ifError(err); - kmsKeyId = keyId; - done(); - } - ); + const bucket = new BucketInfo('enc-bucket-test', 'OwnerId', 'OwnerDisplayName', new Date().toJSON()); + kms.createBucketKey(bucket, log, (err, { masterKeyArn: keyId }) => { + assert.ifError(err); + kmsKeyId = keyId; + done(); + }); }); beforeEach(async () => { @@ -177,30 +175,30 @@ describe('per object encryption headers', () => { putEncryptedObject(s3, bucket, object, target, kmsKeyId, (error, putResp) => { assert.ifError(error); if (target.algo) { - assert.strictEqual(putResp.ServerSideEncryption, target.algo, - 'PutObject response should include ServerSideEncryption header'); + assert.strictEqual( + putResp.ServerSideEncryption, + target.algo, + 'PutObject response should include ServerSideEncryption header', + ); if (target.algo === 'aws:kms') { - assert(putResp.SSEKMSKeyId, - 'PutObject response should include SSEKMSKeyId for aws:kms'); + assert( + putResp.SSEKMSKeyId, + 'PutObject response should include SSEKMSKeyId for aws:kms', + ); } } - return getSSEConfig( - s3, - bucket, - object, - (error, sseConfig) => { - assert.ifError(error); - const expected = createExpected(target, kmsKeyId); - // We differ from aws behavior and always return a - // masterKeyId even when not explicitly configured. - if (expected.algo === 'aws:kms' && !expected.masterKeyId) { - // eslint-disable-next-line no-param-reassign - delete sseConfig.masterKeyId; - } - assert.deepStrictEqual(sseConfig, expected); - done(); + return getSSEConfig(s3, bucket, object, (error, sseConfig) => { + assert.ifError(error); + const expected = createExpected(target, kmsKeyId); + // We differ from aws behavior and always return a + // masterKeyId even when not explicitly configured. + if (expected.algo === 'aws:kms' && !expected.masterKeyId) { + // eslint-disable-next-line no-param-reassign + delete sseConfig.masterKeyId; } - ); + assert.deepStrictEqual(sseConfig, expected); + done(); + }); })); it('should put two encrypted objects in a unencrypted bucket, reusing the generated config', done => @@ -223,48 +221,52 @@ describe('per object encryption headers', () => { } res.forEach(sseConfig => assert.deepStrictEqual(sseConfig, expected)); done(); - } + }, ); - } + }, )); - testCases - .forEach(existing => { + testCases.forEach(existing => { const hasKey = target.masterKeyId ? 'a' : 'no'; const { algo } = target; - it('should override bucket encryption settings with ' - + `algo ${algo || 'none'} with ${hasKey} key id`, done => { - const _existing = Object.assign({}, existing); - if (existing.masterKeyId) { - _existing.masterKeyId = kmsKeyId; - } - const params = { - Bucket: bucket, - ServerSideEncryptionConfiguration: hydrateSSEConfig(_existing), - }; - // no op putBucketEncryption for the unencrypted case - const s3Op = existing.algo ? - (params, cb) => putBucketEncryption(s3, params, cb) : s3NoOp; - s3Op(params, error => { - assert.ifError(error); - return putEncryptedObject(s3, bucket, object, target, kmsKeyId, (error, putResp) => { + it( + 'should override bucket encryption settings with ' + + `algo ${algo || 'none'} with ${hasKey} key id`, + done => { + const _existing = Object.assign({}, existing); + if (existing.masterKeyId) { + _existing.masterKeyId = kmsKeyId; + } + const params = { + Bucket: bucket, + ServerSideEncryptionConfiguration: hydrateSSEConfig(_existing), + }; + // no op putBucketEncryption for the unencrypted case + const s3Op = existing.algo ? (params, cb) => putBucketEncryption(s3, params, cb) : s3NoOp; + s3Op(params, error => { assert.ifError(error); - if (target.algo) { - assert.strictEqual(putResp.ServerSideEncryption, target.algo, - 'PutObject response should include ServerSideEncryption header'); - if (target.algo === 'aws:kms') { - assert(putResp.SSEKMSKeyId, - 'PutObject response should include SSEKMSKeyId for aws:kms'); + return putEncryptedObject(s3, bucket, object, target, kmsKeyId, (error, putResp) => { + assert.ifError(error); + if (target.algo) { + assert.strictEqual( + putResp.ServerSideEncryption, + target.algo, + 'PutObject response should include ServerSideEncryption header', + ); + if (target.algo === 'aws:kms') { + assert( + putResp.SSEKMSKeyId, + 'PutObject response should include SSEKMSKeyId for aws:kms', + ); + } + } else if (existing.algo) { + assert.strictEqual( + putResp.ServerSideEncryption, + existing.algo, + 'PutObject response should include ServerSideEncryption from bucket default', + ); } - } else if (existing.algo) { - assert.strictEqual(putResp.ServerSideEncryption, existing.algo, - 'PutObject response should include ServerSideEncryption from bucket default'); - } - return getSSEConfig( - s3, - bucket, - object, - (error, sseConfig) => { + return getSSEConfig(s3, bucket, object, (error, sseConfig) => { assert.ifError(error); let expected = createExpected(target, kmsKeyId); // In the null case the expected encryption config is @@ -280,29 +282,25 @@ describe('per object encryption headers', () => { } assert.deepStrictEqual(sseConfig, expected); done(); - } - ); + }); + }); }); - }); - }); + }, + ); }); - testCases - .forEach(existing => it('should copy an object to an encrypted key overriding bucket settings', - done => { + testCases.forEach(existing => + it('should copy an object to an encrypted key overriding bucket settings', done => { const _existing = Object.assign({}, existing); if (existing.masterKeyId) { - _existing.masterKeyId = config.kmsHideScalityArn - ? getKeyIdFromArn(kmsKeyId) - : kmsKeyId; + _existing.masterKeyId = config.kmsHideScalityArn ? getKeyIdFromArn(kmsKeyId) : kmsKeyId; } const params = { Bucket: bucket2, ServerSideEncryptionConfiguration: hydrateSSEConfig(_existing), }; // no op putBucketEncryption for the unencrypted case - const s3Op = existing.algo ? - (params, cb) => putBucketEncryption(s3, params, cb) : s3NoOp; + const s3Op = existing.algo ? (params, cb) => putBucketEncryption(s3, params, cb) : s3NoOp; s3Op(params, error => { assert.ifError(error); return putEncryptedObject(s3, bucket, object, target, kmsKeyId, error => { @@ -320,32 +318,28 @@ describe('per object encryption headers', () => { } return copyObject(s3, copyParams, error => { assert.ifError(error); - return getSSEConfig( - s3, - bucket2, - object2, - (error, sseConfig) => { - assert.ifError(error); - let expected = createExpected(target, kmsKeyId); - // In the null case the expected encryption config is - // the buckets default policy - if (!target.algo) { - expected = _existing; - } - // We differ from aws behavior and always return a - // masterKeyId even when not explicitly configured. - if (expected.algo === 'aws:kms' && !expected.masterKeyId) { + return getSSEConfig(s3, bucket2, object2, (error, sseConfig) => { + assert.ifError(error); + let expected = createExpected(target, kmsKeyId); + // In the null case the expected encryption config is + // the buckets default policy + if (!target.algo) { + expected = _existing; + } + // We differ from aws behavior and always return a + // masterKeyId even when not explicitly configured. + if (expected.algo === 'aws:kms' && !expected.masterKeyId) { // eslint-disable-next-line no-param-reassign - delete sseConfig.masterKeyId; - } - assert.deepStrictEqual(sseConfig, expected); - done(); + delete sseConfig.masterKeyId; } - ); + assert.deepStrictEqual(sseConfig, expected); + done(); + }); }); }); }); - })); + }), + ); it('should init an encrypted MPU and put an encrypted part', done => { const params = { diff --git a/tests/functional/aws-node-sdk/test/object/get.js b/tests/functional/aws-node-sdk/test/object/get.js index 5b2c7102d0..c3d96f5fac 100644 --- a/tests/functional/aws-node-sdk/test/object/get.js +++ b/tests/functional/aws-node-sdk/test/object/get.js @@ -42,8 +42,7 @@ const etag = `"${etagTrim}"`; const partSize = 1024 * 1024 * 5; // 5MB minumum required part size. function checkNoError(err) { - assert.equal(err, null, - `Expected success, got error ${JSON.stringify(err)}`); + assert.equal(err, null, `Expected success, got error ${JSON.stringify(err)}`); } function checkError(err, code) { @@ -71,145 +70,202 @@ describe('GET object', () => { let s3; function requestGet(fields, cb) { - s3.send(new GetObjectCommand(Object.assign({ - Bucket: bucketName, - Key: objectName, - }, fields))).then(data => cb(null, data)).catch(err => { - if (err.$metadata.httpStatusCode === 304) { - const notModifiedError = new Error('NotModified'); - notModifiedError.name = 'NotModified'; - notModifiedError.$metadata = err.$metadata; - return cb(notModifiedError); - } - return cb(err); - }); + s3.send( + new GetObjectCommand( + Object.assign( + { + Bucket: bucketName, + Key: objectName, + }, + fields, + ), + ), + ) + .then(data => cb(null, data)) + .catch(err => { + if (err.$metadata.httpStatusCode === 304) { + const notModifiedError = new Error('NotModified'); + notModifiedError.name = 'NotModified'; + notModifiedError.$metadata = err.$metadata; + return cb(notModifiedError); + } + return cb(err); + }); } const requestGetPromise = promisify(requestGet); function checkGetObjectPart(key, partNumber, len, body, cb) { - s3.send(new GetObjectCommand({ - Bucket: bucketName, - Key: key, - PartNumber: partNumber, - })).then(async data => { - checkIntegerHeader(data.ContentLength, len); - const md5Hash = crypto.createHash('md5'); - const md5HashExpected = crypto.createHash('md5'); - const bodyText = await data.Body.transformToString(); - assert.strictEqual( - md5Hash.update(bodyText).digest('hex'), - md5HashExpected.update(body).digest('hex') - ); - return cb(); - }).catch(cb); + s3.send( + new GetObjectCommand({ + Bucket: bucketName, + Key: key, + PartNumber: partNumber, + }), + ) + .then(async data => { + checkIntegerHeader(data.ContentLength, len); + const md5Hash = crypto.createHash('md5'); + const md5HashExpected = crypto.createHash('md5'); + const bodyText = await data.Body.transformToString(); + assert.strictEqual( + md5Hash.update(bodyText).digest('hex'), + md5HashExpected.update(body).digest('hex'), + ); + return cb(); + }) + .catch(cb); } // Upload parts with the given partNumbers array and complete MPU. function completeMPU(partNumbers, cb) { let ETags = []; - return async.waterfall([ - next => { - const createMpuParams = { - Bucket: bucketName, - Key: objectName, - }; - - s3.send(new CreateMultipartUploadCommand(createMpuParams)).then(data => - next(null, data.UploadId)).catch(next); - }, - (uploadId, next) => - async.eachSeries(partNumbers, (partNumber, callback) => { - const uploadPartParams = { + return async.waterfall( + [ + next => { + const createMpuParams = { Bucket: bucketName, Key: objectName, - PartNumber: partNumber, - UploadId: uploadId, - Body: Buffer.alloc(partSize).fill(partNumber), }; - return s3.send(new UploadPartCommand(uploadPartParams)).then(data => { - ETags = ETags.concat(data.ETag); - return callback(); - }).catch(callback); - }, err => next(err, uploadId)), - (uploadId, next) => { - const parts = Array.from(Array(partNumbers.length).keys()); - const params = { - Bucket: bucketName, - Key: objectName, - MultipartUpload: { - Parts: parts.map(n => ({ - ETag: ETags[n], - PartNumber: partNumbers[n], - })), - }, - UploadId: uploadId, - }; - return s3.send(new CompleteMultipartUploadCommand(params)).then(() => - next(null, uploadId)).catch(next); - }, - ], (err, uploadId) => { - if (err) { - if (uploadId) { - return s3.send(new AbortMultipartUploadCommand({ + + s3.send(new CreateMultipartUploadCommand(createMpuParams)) + .then(data => next(null, data.UploadId)) + .catch(next); + }, + (uploadId, next) => + async.eachSeries( + partNumbers, + (partNumber, callback) => { + const uploadPartParams = { + Bucket: bucketName, + Key: objectName, + PartNumber: partNumber, + UploadId: uploadId, + Body: Buffer.alloc(partSize).fill(partNumber), + }; + return s3 + .send(new UploadPartCommand(uploadPartParams)) + .then(data => { + ETags = ETags.concat(data.ETag); + return callback(); + }) + .catch(callback); + }, + err => next(err, uploadId), + ), + (uploadId, next) => { + const parts = Array.from(Array(partNumbers.length).keys()); + const params = { Bucket: bucketName, Key: objectName, + MultipartUpload: { + Parts: parts.map(n => ({ + ETag: ETags[n], + PartNumber: partNumbers[n], + })), + }, UploadId: uploadId, - })).then(() => cb(err)).catch(() => cb(err)); + }; + return s3 + .send(new CompleteMultipartUploadCommand(params)) + .then(() => next(null, uploadId)) + .catch(next); + }, + ], + (err, uploadId) => { + if (err) { + if (uploadId) { + return s3 + .send( + new AbortMultipartUploadCommand({ + Bucket: bucketName, + Key: objectName, + UploadId: uploadId, + }), + ) + .then(() => cb(err)) + .catch(() => cb(err)); + } + return cb(err); } - return cb(err); - } - return cb(); - }); + return cb(); + }, + ); } function createMPUAndPutTwoParts(partTwoBody, cb) { let uploadId; const ETags = []; - return async.waterfall([ - next => s3.send(new CreateMultipartUploadCommand({ - Bucket: bucketName, - Key: copyPartKey, - })).then(data => { - uploadId = data.UploadId; - return next(); - }).catch(next), - // Copy an object with three parts. - next => s3.send(new UploadPartCopyCommand({ - Bucket: bucketName, - CopySource: `/${bucketName}/${objectName}`, - Key: copyPartKey, - PartNumber: 1, - UploadId: uploadId, - })).then(data => { - ETags[0] = data.CopyPartResult.ETag; - return next(); - }).catch(next), - // Put an object with one part. - next => s3.send(new UploadPartCommand({ - Bucket: bucketName, - Key: copyPartKey, - PartNumber: 2, - UploadId: uploadId, - Body: partTwoBody, - })).then(data => { - ETags[1] = data.ETag; - return next(); - }).catch(next), - ], err => { - if (err) { - if (uploadId) { - return s3.send(new AbortMultipartUploadCommand({ - Bucket: bucketName, - Key: copyPartKey, - UploadId: uploadId, - })).then(() => cb(err)).catch(() => cb(err)); + return async.waterfall( + [ + next => + s3 + .send( + new CreateMultipartUploadCommand({ + Bucket: bucketName, + Key: copyPartKey, + }), + ) + .then(data => { + uploadId = data.UploadId; + return next(); + }) + .catch(next), + // Copy an object with three parts. + next => + s3 + .send( + new UploadPartCopyCommand({ + Bucket: bucketName, + CopySource: `/${bucketName}/${objectName}`, + Key: copyPartKey, + PartNumber: 1, + UploadId: uploadId, + }), + ) + .then(data => { + ETags[0] = data.CopyPartResult.ETag; + return next(); + }) + .catch(next), + // Put an object with one part. + next => + s3 + .send( + new UploadPartCommand({ + Bucket: bucketName, + Key: copyPartKey, + PartNumber: 2, + UploadId: uploadId, + Body: partTwoBody, + }), + ) + .then(data => { + ETags[1] = data.ETag; + return next(); + }) + .catch(next), + ], + err => { + if (err) { + if (uploadId) { + return s3 + .send( + new AbortMultipartUploadCommand({ + Bucket: bucketName, + Key: copyPartKey, + UploadId: uploadId, + }), + ) + .then(() => cb(err)) + .catch(() => cb(err)); + } + return cb(err); } - return cb(err); - } - return cb(null, uploadId, ETags); - }); + return cb(null, uploadId, ETags); + }, + ); } before(async () => { @@ -220,98 +276,96 @@ describe('GET object', () => { }); after(async () => { - await s3.send(new DeleteObjectCommand({ Bucket: bucketName, Key: objectName })); - await s3.send(new DeleteBucketCommand({ Bucket: bucketName })); + await s3.send(new DeleteObjectCommand({ Bucket: bucketName, Key: objectName })); + await s3.send(new DeleteBucketCommand({ Bucket: bucketName })); }); - it('should return NoSuchKey error when no such object', - done => { - s3.send(new GetObjectCommand({ Bucket: bucketName, Key: 'nope' })).then(() => { + it('should return NoSuchKey error when no such object', done => { + s3.send(new GetObjectCommand({ Bucket: bucketName, Key: 'nope' })) + .then(() => { assert.fail('Expected failure but got success'); - }).catch(err => { + }) + .catch(err => { assert.strictEqual(err.name, 'NoSuchKey'); return done(); }); - }); + }); - it('should return NoSuchKey error when no such object even with key longer than 915 bytes', - done => { - s3.send(new GetObjectCommand({ Bucket: bucketName, Key: 'a'.repeat(2000) })).then(() => { + it('should return NoSuchKey error when no such object even with key longer than 915 bytes', done => { + s3.send(new GetObjectCommand({ Bucket: bucketName, Key: 'a'.repeat(2000) })) + .then(() => { assert.fail('Expected failure but got success'); - }).catch(err => { + }) + .catch(err => { assert.strictEqual(err.name, 'NoSuchKey'); return done(); }); - }); + }); - describe('Additional headers: [Cache-Control, Content-Disposition, ' + - 'Content-Encoding, Expires, Accept-Ranges]', () => { - describe('if specified in put object request', () => { - before(async () => { - const params = { - Bucket: bucketName, - Key: objectName, - CacheControl: cacheControl, - ContentDisposition: contentDisposition, - ContentEncoding: contentEncoding, - ContentType: contentType, - Expires: expires, - }; - await s3.send(new PutObjectCommand(params)); - }); - it('should return additional headers', done => { - s3.send(new GetObjectCommand({ Bucket: bucketName, Key: objectName })).then(res => { - assert.strictEqual(res.CacheControl, - cacheControl); - assert.strictEqual(res.ContentDisposition, - contentDisposition); - // Should remove V4 streaming value 'aws-chunked' - // to be compatible with AWS behavior - assert.strictEqual(res.ContentEncoding, - 'gzip'); - assert.strictEqual(res.ContentType, contentType); - assert.strictEqual(res.Expires.toGMTString(), - new Date(expires).toGMTString()); - assert.strictEqual(res.AcceptRanges, 'bytes'); - return done(); - }).catch(done); + describe( + 'Additional headers: [Cache-Control, Content-Disposition, ' + 'Content-Encoding, Expires, Accept-Ranges]', + () => { + describe('if specified in put object request', () => { + before(async () => { + const params = { + Bucket: bucketName, + Key: objectName, + CacheControl: cacheControl, + ContentDisposition: contentDisposition, + ContentEncoding: contentEncoding, + ContentType: contentType, + Expires: expires, + }; + await s3.send(new PutObjectCommand(params)); + }); + it('should return additional headers', done => { + s3.send(new GetObjectCommand({ Bucket: bucketName, Key: objectName })) + .then(res => { + assert.strictEqual(res.CacheControl, cacheControl); + assert.strictEqual(res.ContentDisposition, contentDisposition); + // Should remove V4 streaming value 'aws-chunked' + // to be compatible with AWS behavior + assert.strictEqual(res.ContentEncoding, 'gzip'); + assert.strictEqual(res.ContentType, contentType); + assert.strictEqual(res.Expires.toGMTString(), new Date(expires).toGMTString()); + assert.strictEqual(res.AcceptRanges, 'bytes'); + return done(); + }) + .catch(done); + }); }); - }); - describe('if response content headers are set in query', () => { - before(async () => { - await s3.send(new PutObjectCommand({ Bucket: bucketName, Key: objectName })); - }); + describe('if response content headers are set in query', () => { + before(async () => { + await s3.send(new PutObjectCommand({ Bucket: bucketName, Key: objectName })); + }); - it('should return additional headers even if not set in ' + - 'put object request', done => { - const params = { - Bucket: bucketName, - Key: objectName, - ResponseCacheControl: cacheControl, - ResponseContentDisposition: contentDisposition, - ResponseContentEncoding: contentEncoding, - ResponseContentLanguage: contentLanguage, - ResponseContentType: contentType, - ResponseExpires: expires, - }; - s3.send(new GetObjectCommand(params)).then(res => { - assert.strictEqual(res.CacheControl, - cacheControl); - assert.strictEqual(res.ContentDisposition, - contentDisposition); - assert.strictEqual(res.ContentEncoding, - contentEncoding); - assert.strictEqual(res.ContentLanguage, - contentLanguage); - assert.strictEqual(res.ContentType, contentType); - assert.strictEqual(res.Expires.toGMTString(), - new Date(expires).toGMTString()); - return done(); - }).catch(done); + it('should return additional headers even if not set in ' + 'put object request', done => { + const params = { + Bucket: bucketName, + Key: objectName, + ResponseCacheControl: cacheControl, + ResponseContentDisposition: contentDisposition, + ResponseContentEncoding: contentEncoding, + ResponseContentLanguage: contentLanguage, + ResponseContentType: contentType, + ResponseExpires: expires, + }; + s3.send(new GetObjectCommand(params)) + .then(res => { + assert.strictEqual(res.CacheControl, cacheControl); + assert.strictEqual(res.ContentDisposition, contentDisposition); + assert.strictEqual(res.ContentEncoding, contentEncoding); + assert.strictEqual(res.ContentLanguage, contentLanguage); + assert.strictEqual(res.ContentType, contentType); + assert.strictEqual(res.Expires.toGMTString(), new Date(expires).toGMTString()); + return done(); + }) + .catch(done); + }); }); - }); - }); + }, + ); describe('x-amz-website-redirect-location header', () => { before(async () => { @@ -322,12 +376,13 @@ describe('GET object', () => { }; await s3.send(new PutObjectCommand(params)); }); - it('should return website redirect header if specified in ' + - 'objectPUT request', done => { - s3.send(new GetObjectCommand({ Bucket: bucketName, Key: objectName })).then(res => { - assert.strictEqual(res.WebsiteRedirectLocation, '/'); - return done(); - }).catch(done); + it('should return website redirect header if specified in ' + 'objectPUT request', done => { + s3.send(new GetObjectCommand({ Bucket: bucketName, Key: objectName })) + .then(res => { + assert.strictEqual(res.WebsiteRedirectLocation, '/'); + return done(); + }) + .catch(done); }); }); @@ -352,27 +407,31 @@ describe('GET object', () => { await s3.send(new PutObjectCommand(params)); }); - it('should not return "x-amz-tagging-count" if no tag ' + - 'associated with the object', - done => { - s3.send(new GetObjectCommand(params)).then(data => { - assert.strictEqual(data.TagCount, undefined); - return done(); - }).catch(done); + it('should not return "x-amz-tagging-count" if no tag ' + 'associated with the object', done => { + s3.send(new GetObjectCommand(params)) + .then(data => { + assert.strictEqual(data.TagCount, undefined); + return done(); + }) + .catch(done); }); describe('tag associated with the object', () => { beforeEach(async () => { await s3.send(new PutObjectTaggingCommand(paramsTagging)); }); - it('should return "x-amz-tagging-count" header that provides ' + - 'the count of number of tags associated with the object', - done => { - s3.send(new GetObjectCommand(params)).then(data => { - assert.equal(data.TagCount, 1); - return done(); - }).catch(done); - }); + it( + 'should return "x-amz-tagging-count" header that provides ' + + 'the count of number of tags associated with the object', + done => { + s3.send(new GetObjectCommand(params)) + .then(data => { + assert.equal(data.TagCount, 1); + return done(); + }) + .catch(done); + }, + ); }); }); @@ -381,43 +440,33 @@ describe('GET object', () => { beforeEach(async () => { await s3.send(new PutObjectCommand(params)); }); - it('If-Match: returns no error when ETag match, with double ' + - 'quotes around ETag', - done => { - requestGet({ IfMatch: etag }, err => { - checkNoError(err); - done(); - }); + it('If-Match: returns no error when ETag match, with double ' + 'quotes around ETag', done => { + requestGet({ IfMatch: etag }, err => { + checkNoError(err); + done(); }); + }); - it('If-Match: returns no error when one of ETags match, with ' + - 'double quotes around ETag', - done => { - requestGet({ IfMatch: - `non-matching,${etag}` }, err => { - checkNoError(err); - done(); - }); + it('If-Match: returns no error when one of ETags match, with ' + 'double quotes around ETag', done => { + requestGet({ IfMatch: `non-matching,${etag}` }, err => { + checkNoError(err); + done(); }); + }); - it('If-Match: returns no error when ETag match, without double ' + - 'quotes around ETag', - done => { - requestGet({ IfMatch: etagTrim }, err => { - checkNoError(err); - done(); - }); + it('If-Match: returns no error when ETag match, without double ' + 'quotes around ETag', done => { + requestGet({ IfMatch: etagTrim }, err => { + checkNoError(err); + done(); }); + }); - it('If-Match: returns no error when one of ETags match, without ' + - 'double quotes around ETag', - done => { - requestGet({ IfMatch: - `non-matching,${etagTrim}` }, err => { - checkNoError(err); - done(); - }); + it('If-Match: returns no error when one of ETags match, without ' + 'double quotes around ETag', done => { + requestGet({ IfMatch: `non-matching,${etagTrim}` }, err => { + checkNoError(err); + done(); }); + }); it('If-Match: returns no error when ETag match with *', done => { requestGet({ IfMatch: '*' }, err => { @@ -426,320 +475,364 @@ describe('GET object', () => { }); }); - it('If-Match: returns PreconditionFailed when ETag does not match', - done => { - requestGet({ + it('If-Match: returns PreconditionFailed when ETag does not match', done => { + requestGet( + { IfMatch: 'non-matching ETag', - }, err => { + }, + err => { checkError(err, 'PreconditionFailed'); done(); - }); - }); + }, + ); + }); - it('If-None-Match: returns no error when ETag does not match', - done => { - requestGet({ IfNoneMatch: 'non-matching' }, err => { - checkNoError(err); - done(); - }); + it('If-None-Match: returns no error when ETag does not match', done => { + requestGet({ IfNoneMatch: 'non-matching' }, err => { + checkNoError(err); + done(); }); + }); - it('If-None-Match: returns no error when all ETags do not match', - done => { - requestGet({ - IfNoneMatch: 'non-matching,' + - 'non-matching-either', - }, err => { + it('If-None-Match: returns no error when all ETags do not match', done => { + requestGet( + { + IfNoneMatch: 'non-matching,' + 'non-matching-either', + }, + err => { checkNoError(err); done(); - }); - }); + }, + ); + }); - it('If-None-Match: returns NotModified when ETag match, with ' + - 'double quotes around ETag', - done => { - requestGet({ IfNoneMatch: etag }, err => { - checkError(err, 'NotModified'); - done(); - }); + it('If-None-Match: returns NotModified when ETag match, with ' + 'double quotes around ETag', done => { + requestGet({ IfNoneMatch: etag }, err => { + checkError(err, 'NotModified'); + done(); }); + }); - it('If-None-Match: returns NotModified when one of ETags match, ' + - 'with double quotes around ETag', + it( + 'If-None-Match: returns NotModified when one of ETags match, ' + 'with double quotes around ETag', done => { - requestGet({ - IfNoneMatch: `non-matching,${etag}`, - }, err => { - checkError(err, 'NotModified'); - done(); - }); - }); + requestGet( + { + IfNoneMatch: `non-matching,${etag}`, + }, + err => { + checkError(err, 'NotModified'); + done(); + }, + ); + }, + ); - it('If-None-Match: returns NotModified when value is "*"', - done => { - requestGet({ + it('If-None-Match: returns NotModified when value is "*"', done => { + requestGet( + { IfNoneMatch: '*', - }, err => { - checkError(err, 'NotModified'); - done(); - }); - }); - - it('If-None-Match: returns NotModified when ETag match, without ' + - 'double quotes around ETag', - done => { - requestGet({ IfNoneMatch: etagTrim }, err => { + }, + err => { checkError(err, 'NotModified'); done(); - }); - }); + }, + ); + }); - it('If-None-Match: returns NotModified when one of ETags match, ' + - 'without double quotes around ETag', - done => { - requestGet({ - IfNoneMatch: `non-matching,${etagTrim}`, - }, err => { - checkError(err, 'NotModified'); - done(); - }); + it('If-None-Match: returns NotModified when ETag match, without ' + 'double quotes around ETag', done => { + requestGet({ IfNoneMatch: etagTrim }, err => { + checkError(err, 'NotModified'); + done(); }); + }); - it('If-Modified-Since: returns no error if Last modified date is ' + - 'greater', + it( + 'If-None-Match: returns NotModified when one of ETags match, ' + 'without double quotes around ETag', done => { - requestGet({ IfModifiedSince: dateFromNow(-1) }, + requestGet( + { + IfNoneMatch: `non-matching,${etagTrim}`, + }, err => { - checkNoError(err); + checkError(err, 'NotModified'); done(); - }); + }, + ); + }, + ); + + it('If-Modified-Since: returns no error if Last modified date is ' + 'greater', done => { + requestGet({ IfModifiedSince: dateFromNow(-1) }, err => { + checkNoError(err); + done(); }); + }); // Skipping this test, because real AWS does not provide error as // expected - it.skip('If-Modified-Since: returns NotModified if Last modified ' + - 'date is lesser', - done => { - requestGet({ IfModifiedSince: dateFromNow(1) }, - err => { - checkError(err, 'NotModified'); - done(); - }); + it.skip('If-Modified-Since: returns NotModified if Last modified ' + 'date is lesser', done => { + requestGet({ IfModifiedSince: dateFromNow(1) }, err => { + checkError(err, 'NotModified'); + done(); }); + }); - it('If-Modified-Since: returns NotModified if Last modified ' + - 'date is equal', - done => { - s3.send(new HeadObjectCommand({ Bucket: bucketName, Key: objectName })).then(data => { + it('If-Modified-Since: returns NotModified if Last modified ' + 'date is equal', done => { + s3.send(new HeadObjectCommand({ Bucket: bucketName, Key: objectName })) + .then(data => { const lastModified = dateConvert(data.LastModified); requestGet({ IfModifiedSince: lastModified }, err => { checkError(err, 'NotModified'); done(); }); - }).catch(done); - }); + }) + .catch(done); + }); - it('If-Unmodified-Since: returns no error when lastModified date ' + - 'is greater', - done => { - requestGet({ IfUnmodifiedSince: dateFromNow(1) }, - err => { - checkNoError(err); - done(); - }); + it('If-Unmodified-Since: returns no error when lastModified date ' + 'is greater', done => { + requestGet({ IfUnmodifiedSince: dateFromNow(1) }, err => { + checkNoError(err); + done(); }); + }); - it('If-Unmodified-Since: returns no error when lastModified ' + - 'date is equal', done => { - s3.send(new HeadObjectCommand({ Bucket: bucketName, Key: objectName })).then(data => { - const lastModified = dateConvert(data.LastModified); - requestGet({ IfUnmodifiedSince: lastModified }, - err => { + it('If-Unmodified-Since: returns no error when lastModified ' + 'date is equal', done => { + s3.send(new HeadObjectCommand({ Bucket: bucketName, Key: objectName })) + .then(data => { + const lastModified = dateConvert(data.LastModified); + requestGet({ IfUnmodifiedSince: lastModified }, err => { checkNoError(err); done(); }); - }).catch(done); + }) + .catch(done); }); - it('If-Unmodified-Since: returns PreconditionFailed when ' + - 'lastModified date is lesser', - done => { - requestGet({ IfUnmodifiedSince: dateFromNow(-1) }, - err => { - checkError(err, 'PreconditionFailed'); - done(); - }); + it('If-Unmodified-Since: returns PreconditionFailed when ' + 'lastModified date is lesser', done => { + requestGet({ IfUnmodifiedSince: dateFromNow(-1) }, err => { + checkError(err, 'PreconditionFailed'); + done(); }); + }); - it('If-Match & If-Unmodified-Since: returns no error when match ' + - 'Etag and lastModified is greater', + it( + 'If-Match & If-Unmodified-Since: returns no error when match ' + 'Etag and lastModified is greater', done => { - requestGet({ + requestGet( + { + IfMatch: etagTrim, + IfUnmodifiedSince: dateFromNow(-1), + }, + err => { + checkNoError(err); + done(); + }, + ); + }, + ); + + it('If-Match match & If-Unmodified-Since match', done => { + requestGet( + { IfMatch: etagTrim, - IfUnmodifiedSince: dateFromNow(-1), - }, err => { + IfUnmodifiedSince: dateFromNow(1), + }, + err => { checkNoError(err); done(); - }); - }); - - it('If-Match match & If-Unmodified-Since match', done => { - requestGet({ - IfMatch: etagTrim, - IfUnmodifiedSince: dateFromNow(1), - }, err => { - checkNoError(err); - done(); - }); + }, + ); }); it('If-Match not match & If-Unmodified-Since not match', done => { - requestGet({ - IfMatch: 'non-matching', - IfUnmodifiedSince: dateFromNow(-1), - }, err => { - checkError(err, 'PreconditionFailed'); - done(); - }); + requestGet( + { + IfMatch: 'non-matching', + IfUnmodifiedSince: dateFromNow(-1), + }, + err => { + checkError(err, 'PreconditionFailed'); + done(); + }, + ); }); it('If-Match not match & If-Unmodified-Since match', done => { - requestGet({ - IfMatch: 'non-matching', - IfUnmodifiedSince: dateFromNow(1), - }, err => { - checkError(err, 'PreconditionFailed'); - done(); - }); + requestGet( + { + IfMatch: 'non-matching', + IfUnmodifiedSince: dateFromNow(1), + }, + err => { + checkError(err, 'PreconditionFailed'); + done(); + }, + ); }); // Skipping this test, because real AWS does not provide error as // expected it.skip('If-Match match & If-Modified-Since not match', done => { - requestGet({ - IfMatch: etagTrim, - IfModifiedSince: dateFromNow(1), - }, err => { - checkNoError(err); - done(); - }); + requestGet( + { + IfMatch: etagTrim, + IfModifiedSince: dateFromNow(1), + }, + err => { + checkNoError(err); + done(); + }, + ); }); it('If-Match match & If-Modified-Since match', done => { - requestGet({ - IfMatch: etagTrim, - IfModifiedSince: dateFromNow(-1), - }, err => { - checkNoError(err); - done(); - }); + requestGet( + { + IfMatch: etagTrim, + IfModifiedSince: dateFromNow(-1), + }, + err => { + checkNoError(err); + done(); + }, + ); }); it('If-Match not match & If-Modified-Since not match', done => { - requestGet({ - IfMatch: 'non-matching', - IfModifiedSince: dateFromNow(1), - }, err => { - checkError(err, 'PreconditionFailed'); - done(); - }); + requestGet( + { + IfMatch: 'non-matching', + IfModifiedSince: dateFromNow(1), + }, + err => { + checkError(err, 'PreconditionFailed'); + done(); + }, + ); }); it('If-Match not match & If-Modified-Since match', done => { - requestGet({ - IfMatch: 'non-matching', - IfModifiedSince: dateFromNow(-1), - }, err => { - checkError(err, 'PreconditionFailed'); - done(); - }); + requestGet( + { + IfMatch: 'non-matching', + IfModifiedSince: dateFromNow(-1), + }, + err => { + checkError(err, 'PreconditionFailed'); + done(); + }, + ); }); - it('If-None-Match & If-Modified-Since: returns NotModified when ' + - 'Etag does not match and lastModified is greater', + it( + 'If-None-Match & If-Modified-Since: returns NotModified when ' + + 'Etag does not match and lastModified is greater', done => { - requestGet({ + requestGet( + { + IfNoneMatch: etagTrim, + IfModifiedSince: dateFromNow(1), + }, + err => { + checkError(err, 'NotModified'); + done(); + }, + ); + }, + ); + + it('If-None-Match not match & If-Modified-Since not match', done => { + requestGet( + { IfNoneMatch: etagTrim, IfModifiedSince: dateFromNow(1), - }, err => { + }, + err => { checkError(err, 'NotModified'); done(); - }); - }); - - it('If-None-Match not match & If-Modified-Since not match', - done => { - requestGet({ - IfNoneMatch: etagTrim, - IfModifiedSince: dateFromNow(1), - }, err => { - checkError(err, 'NotModified'); - done(); - }); + }, + ); }); it('If-None-Match match & If-Modified-Since match', done => { - requestGet({ - IfNoneMatch: 'non-matching', - IfModifiedSince: dateFromNow(-1), - }, err => { - checkNoError(err); - done(); - }); + requestGet( + { + IfNoneMatch: 'non-matching', + IfModifiedSince: dateFromNow(-1), + }, + err => { + checkNoError(err); + done(); + }, + ); }); // Skipping this test, because real AWS does not provide error as // expected - it.skip('If-None-Match match & If-Modified-Since not match', - done => { - requestGet({ - IfNoneMatch: 'non-matching', - IfModifiedSince: dateFromNow(1), - }, err => { - checkError(err, 'PreconditionFailed'); - done(); - }); + it.skip('If-None-Match match & If-Modified-Since not match', done => { + requestGet( + { + IfNoneMatch: 'non-matching', + IfModifiedSince: dateFromNow(1), + }, + err => { + checkError(err, 'PreconditionFailed'); + done(); + }, + ); }); it('If-None-Match match & If-Unmodified-Since match', done => { - requestGet({ - IfNoneMatch: 'non-matching', - IfUnmodifiedSince: dateFromNow(1), - }, err => { - checkNoError(err); - done(); - }); + requestGet( + { + IfNoneMatch: 'non-matching', + IfUnmodifiedSince: dateFromNow(1), + }, + err => { + checkNoError(err); + done(); + }, + ); }); it('If-None-Match match & If-Unmodified-Since not match', done => { - requestGet({ - IfNoneMatch: 'non-matching', - IfUnmodifiedSince: dateFromNow(-1), - }, err => { - checkError(err, 'PreconditionFailed'); - done(); - }); + requestGet( + { + IfNoneMatch: 'non-matching', + IfUnmodifiedSince: dateFromNow(-1), + }, + err => { + checkError(err, 'PreconditionFailed'); + done(); + }, + ); }); it('If-None-Match not match & If-Unmodified-Since match', done => { - requestGet({ - IfNoneMatch: etagTrim, - IfUnmodifiedSince: dateFromNow(1), - }, err => { - checkError(err, 'NotModified'); - done(); - }); + requestGet( + { + IfNoneMatch: etagTrim, + IfUnmodifiedSince: dateFromNow(1), + }, + err => { + checkError(err, 'NotModified'); + done(); + }, + ); }); - it('If-None-Match not match & If-Unmodified-Since not match', - done => { - requestGet({ - IfNoneMatch: etagTrim, - IfUnmodifiedSince: dateFromNow(-1), - }, err => { - checkError(err, 'PreconditionFailed'); - done(); - }); + it('If-None-Match not match & If-Unmodified-Since not match', done => { + requestGet( + { + IfNoneMatch: etagTrim, + IfUnmodifiedSince: dateFromNow(-1), + }, + err => { + checkError(err, 'PreconditionFailed'); + done(); + }, + ); }); }); @@ -749,8 +842,8 @@ describe('GET object', () => { const invalidPartNumbers = [-1, 0, 10001]; orderedPartNumbers.forEach(num => - it(`should get the body of part ${num} when ordered MPU`, - done => completeMPU(orderedPartNumbers, err => { + it(`should get the body of part ${num} when ordered MPU`, done => + completeMPU(orderedPartNumbers, err => { checkNoError(err); return requestGet({ PartNumber: num }, async (err, data) => { checkNoError(err); @@ -761,135 +854,143 @@ describe('GET object', () => { const bodyText = await data.Body.transformToString(); assert.strictEqual( md5Hash.update(bodyText).digest('hex'), - md5HashExpected.update(expected).digest('hex') + md5HashExpected.update(expected).digest('hex'), ); return done(); }); - }))); + })), + ); // Use the orderedPartNumbers to retrieve parts with GetObject. orderedPartNumbers.forEach(num => - it(`should get the body of part ${num} when unordered MPU`, - done => completeMPU(unOrderedPartNumbers, err => { + it(`should get the body of part ${num} when unordered MPU`, done => + completeMPU(unOrderedPartNumbers, err => { checkNoError(err); return requestGet({ PartNumber: num }, async (err, data) => { checkNoError(err); checkIntegerHeader(data.ContentLength, partSize); const md5Hash = crypto.createHash('md5'); const md5HashExpected = crypto.createHash('md5'); - const expected = Buffer.alloc(partSize) - .fill(unOrderedPartNumbers[num - 1]); + const expected = Buffer.alloc(partSize).fill(unOrderedPartNumbers[num - 1]); const bodyText = await data.Body.transformToString(); assert.strictEqual( md5Hash.update(bodyText).digest('hex'), - md5HashExpected.update(expected).digest('hex') + md5HashExpected.update(expected).digest('hex'), ); return done(); }); - }))); + })), + ); invalidPartNumbers.forEach(num => - it(`should not accept a partNumber that is not 1-10000: ${num}`, - done => completeMPU(orderedPartNumbers, err => { - checkNoError(err); - return requestGet({ PartNumber: num }, err => { - checkError(err, 'InvalidArgument'); - done(); - }); - }))); + it(`should not accept a partNumber that is not 1-10000: ${num}`, done => + completeMPU(orderedPartNumbers, err => { + checkNoError(err); + return requestGet({ PartNumber: num }, err => { + checkError(err, 'InvalidArgument'); + done(); + }); + })), + ); - it('should not accept a part number greater than the total parts ' + - 'uploaded for an MPU', done => + it('should not accept a part number greater than the total parts ' + 'uploaded for an MPU', done => completeMPU(orderedPartNumbers, err => { checkNoError(err); return requestGet({ PartNumber: 11 }, err => { checkError(err, 'InvalidPartNumber'); done(); }); - })); + }), + ); - it('should accept a part number of 1 for regular put object', - async () => { - await s3.send(new PutObjectCommand({ + it('should accept a part number of 1 for regular put object', async () => { + await s3.send( + new PutObjectCommand({ Bucket: bucketName, Key: objectName, Body: Buffer.alloc(10), - })); - - const data = await requestGetPromise({ PartNumber: 1 }); - const md5Hash = crypto.createHash('md5'); - const md5HashExpected = crypto.createHash('md5'); - const expected = Buffer.alloc(10).fill(0); - const bodyText = await data.Body.transformToString(); - assert.strictEqual( - md5Hash.update(bodyText).digest('hex'), - md5HashExpected.update(expected).digest('hex') - ); - }); + }), + ); + + const data = await requestGetPromise({ PartNumber: 1 }); + const md5Hash = crypto.createHash('md5'); + const md5HashExpected = crypto.createHash('md5'); + const expected = Buffer.alloc(10).fill(0); + const bodyText = await data.Body.transformToString(); + assert.strictEqual( + md5Hash.update(bodyText).digest('hex'), + md5HashExpected.update(expected).digest('hex'), + ); + }); it('should accept a part number that is a string', async () => { - await s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName, - Body: Buffer.alloc(10), - })); - + await s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + Body: Buffer.alloc(10), + }), + ); + const data = await requestGetPromise({ PartNumber: '1' }); - checkIntegerHeader(data.ContentLength, 10); + checkIntegerHeader(data.ContentLength, 10); const md5Hash = crypto.createHash('md5'); const md5HashExpected = crypto.createHash('md5'); const expected = Buffer.alloc(10).fill(0); const bodyText = await data.Body.transformToString(); assert.strictEqual( md5Hash.update(bodyText).digest('hex'), - md5HashExpected.update(expected).digest('hex') + md5HashExpected.update(expected).digest('hex'), ); }); - it('should not accept a part number greater than 1 for regular ' + - 'put object', async () => { - await s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName, - Body: Buffer.alloc(10), - })); - + it('should not accept a part number greater than 1 for regular ' + 'put object', async () => { + await s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + Body: Buffer.alloc(10), + }), + ); + await assert.rejects( - () => requestGetPromise({ PartNumber: 2 }), - err => { - checkError(err, 'InvalidPartNumber'); - return true; - } - ); + () => requestGetPromise({ PartNumber: 2 }), + err => { + checkError(err, 'InvalidPartNumber'); + return true; + }, + ); }); it('should not accept both PartNumber and Range as params', done => completeMPU(orderedPartNumbers, err => { checkNoError(err); - return requestGet({ - PartNumber: 1, - Range: 'bytes=0-10', - }, err => { - checkError(err, 'InvalidRequest'); - done(); - }); + return requestGet( + { + PartNumber: 1, + Range: 'bytes=0-10', + }, + err => { + checkError(err, 'InvalidRequest'); + done(); + }, + ); })); - it('should not include PartsCount response header for regular ' + - 'put object', async () => { - await s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName, - Body: Buffer.alloc(10), - })); - + it('should not include PartsCount response header for regular ' + 'put object', async () => { + await s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + Body: Buffer.alloc(10), + }), + ); + const data = await requestGetPromise({ PartNumber: 1 }); - assert.strictEqual('PartsCount' in data, false, - 'PartsCount header is present.'); + assert.strictEqual('PartsCount' in data, false, 'PartsCount header is present.'); }); - it('should include PartsCount response header for mpu object', - done => { + it('should include PartsCount response header for mpu object', done => { completeMPU(orderedPartNumbers, err => { assert.ifError(err); return requestGet({ PartNumber: 1 }, (err, data) => { @@ -903,123 +1004,151 @@ describe('GET object', () => { describe('uploadPartCopy', () => { // The original object was composed of three parts const partOneSize = partSize * 10; - const bufs = orderedPartNumbers.map(n => - Buffer.alloc(partSize, n)); + const bufs = orderedPartNumbers.map(n => Buffer.alloc(partSize, n)); const partOneBody = Buffer.concat(bufs, partOneSize); const partTwoBody = Buffer.alloc(partSize, 4); - beforeEach(done => async.waterfall([ - next => completeMPU(orderedPartNumbers, next), - next => createMPUAndPutTwoParts(partTwoBody, next), - (uploadId, ETags, next) => - s3.send(new CompleteMultipartUploadCommand({ - Bucket: bucketName, - Key: copyPartKey, - MultipartUpload: { - Parts: [ - { - ETag: ETags[0], - PartNumber: 1, - }, - { - ETag: ETags[1], - PartNumber: 2, - }, - ], - }, - UploadId: uploadId, - })).then(() => next()).catch(next), - ], done)); + beforeEach(done => + async.waterfall( + [ + next => completeMPU(orderedPartNumbers, next), + next => createMPUAndPutTwoParts(partTwoBody, next), + (uploadId, ETags, next) => + s3 + .send( + new CompleteMultipartUploadCommand({ + Bucket: bucketName, + Key: copyPartKey, + MultipartUpload: { + Parts: [ + { + ETag: ETags[0], + PartNumber: 1, + }, + { + ETag: ETags[1], + PartNumber: 2, + }, + ], + }, + UploadId: uploadId, + }), + ) + .then(() => next()) + .catch(next), + ], + done, + ), + ); afterEach(async () => { - await s3.send(new DeleteObjectCommand({ - Bucket: bucketName, - Key: copyPartKey, - })); + await s3.send( + new DeleteObjectCommand({ + Bucket: bucketName, + Key: copyPartKey, + }), + ); }); it('should retrieve a part copied from an MPU', done => - checkGetObjectPart(copyPartKey, 1, partOneSize, partOneBody, - done)); + checkGetObjectPart(copyPartKey, 1, partOneSize, partOneBody, done)); - it('should retrieve a part put after part copied from MPU', - done => checkGetObjectPart(copyPartKey, 2, partSize, - partTwoBody, done)); + it('should retrieve a part put after part copied from MPU', done => + checkGetObjectPart(copyPartKey, 2, partSize, partTwoBody, done)); }); describe('uploadPartCopy overwrite', () => { const partOneBody = Buffer.alloc(partSize, 1); // The original object was composed of three parts const partTwoSize = partSize * 10; - const bufs = orderedPartNumbers.map(n => - Buffer.alloc(partSize, n)); + const bufs = orderedPartNumbers.map(n => Buffer.alloc(partSize, n)); const partTwoBody = Buffer.concat(bufs, partTwoSize); - beforeEach(done => async.waterfall([ - next => completeMPU(orderedPartNumbers, next), - next => createMPUAndPutTwoParts(partTwoBody, next), - /* eslint-disable no-param-reassign */ - // Overwrite part one. - (uploadId, ETags, next) => - s3.send(new UploadPartCommand({ - Bucket: bucketName, - Key: copyPartKey, - PartNumber: 1, - UploadId: uploadId, - Body: partOneBody, - })).then(data => { - ETags[0] = data.ETag; - return next(null, uploadId, ETags); - }).catch(next), - // Overwrite part one with an three-part object. - (uploadId, ETags, next) => - s3.send(new UploadPartCopyCommand({ - Bucket: bucketName, - CopySource: `/${bucketName}/${objectName}`, - Key: copyPartKey, - PartNumber: 2, - UploadId: uploadId, - })).then(data => { - ETags[1] = data.CopyPartResult.ETag; - return next(null, uploadId, ETags); - }).catch(next), - /* eslint-enable no-param-reassign */ - (uploadId, ETags, next) => - s3.send(new CompleteMultipartUploadCommand({ - Bucket: bucketName, - Key: copyPartKey, - MultipartUpload: { - Parts: [ - { - ETag: ETags[0], - PartNumber: 1, - }, - { - ETag: ETags[1], - PartNumber: 2, - }, - ], - }, - UploadId: uploadId, - })).then(() => next()).catch(next), - ], done)); + beforeEach(done => + async.waterfall( + [ + next => completeMPU(orderedPartNumbers, next), + next => createMPUAndPutTwoParts(partTwoBody, next), + /* eslint-disable no-param-reassign */ + // Overwrite part one. + (uploadId, ETags, next) => + s3 + .send( + new UploadPartCommand({ + Bucket: bucketName, + Key: copyPartKey, + PartNumber: 1, + UploadId: uploadId, + Body: partOneBody, + }), + ) + .then(data => { + ETags[0] = data.ETag; + return next(null, uploadId, ETags); + }) + .catch(next), + // Overwrite part one with an three-part object. + (uploadId, ETags, next) => + s3 + .send( + new UploadPartCopyCommand({ + Bucket: bucketName, + CopySource: `/${bucketName}/${objectName}`, + Key: copyPartKey, + PartNumber: 2, + UploadId: uploadId, + }), + ) + .then(data => { + ETags[1] = data.CopyPartResult.ETag; + return next(null, uploadId, ETags); + }) + .catch(next), + /* eslint-enable no-param-reassign */ + (uploadId, ETags, next) => + s3 + .send( + new CompleteMultipartUploadCommand({ + Bucket: bucketName, + Key: copyPartKey, + MultipartUpload: { + Parts: [ + { + ETag: ETags[0], + PartNumber: 1, + }, + { + ETag: ETags[1], + PartNumber: 2, + }, + ], + }, + UploadId: uploadId, + }), + ) + .then(() => next()) + .catch(next), + ], + done, + ), + ); afterEach(async () => { - await s3.send(new DeleteObjectCommand({ - Bucket: bucketName, - Key: copyPartKey, - })); + await s3.send( + new DeleteObjectCommand({ + Bucket: bucketName, + Key: copyPartKey, + }), + ); }); - it('should retrieve a part that overwrote another part ' + - 'originally copied from an MPU', done => - checkGetObjectPart(copyPartKey, 1, partSize, partOneBody, - done)); + it('should retrieve a part that overwrote another part ' + 'originally copied from an MPU', done => + checkGetObjectPart(copyPartKey, 1, partSize, partOneBody, done), + ); - it('should retrieve a part copied from an MPU after the ' + - 'original part was overwritten', - done => checkGetObjectPart(copyPartKey, 2, partTwoSize, - partTwoBody, done)); + it('should retrieve a part copied from an MPU after the ' + 'original part was overwritten', done => + checkGetObjectPart(copyPartKey, 2, partTwoSize, partTwoBody, done), + ); }); }); @@ -1031,19 +1160,18 @@ describe('GET object', () => { }; await s3.send(new PutObjectCommand(params)); }); - it('should return website redirect header if specified in ' + - 'objectPUT request', done => { - s3.send(new GetObjectCommand({ Bucket: bucketName, Key: objectName })).then(res => { - assert.strictEqual(res.WebsiteRedirectLocation, - undefined); - return done(); - }).catch(done); + it('should return website redirect header if specified in ' + 'objectPUT request', done => { + s3.send(new GetObjectCommand({ Bucket: bucketName, Key: objectName })) + .then(res => { + assert.strictEqual(res.WebsiteRedirectLocation, undefined); + return done(); + }) + .catch(done); }); }); }); }); - describe('GET object with object lock', () => { withV4(sigCfg => { const bucketUtil = new BucketUtility('default', sigCfg); @@ -1063,59 +1191,69 @@ describe('GET object with object lock', () => { ObjectLockMode: mockMode, ObjectLockLegalHoldStatus: 'ON', }; - return s3.send(new CreateBucketCommand({ - Bucket: bucket, - ObjectLockEnabledForBucket: true, - })) - .then(() => s3.send(new PutObjectCommand(params))) - .then(() => s3.send(new GetObjectCommand({ Bucket: bucket, Key: key }))) - /* eslint-disable no-return-assign */ - .then(res => versionId = res.VersionId) - .catch(err => { - process.stdout.write('Error in before\n'); - throw err; - }); + return ( + s3 + .send( + new CreateBucketCommand({ + Bucket: bucket, + ObjectLockEnabledForBucket: true, + }), + ) + .then(() => s3.send(new PutObjectCommand(params))) + .then(() => s3.send(new GetObjectCommand({ Bucket: bucket, Key: key }))) + /* eslint-disable no-return-assign */ + .then(res => (versionId = res.VersionId)) + .catch(err => { + process.stdout.write('Error in before\n'); + throw err; + }) + ); }); - afterEach(() => changeLockPromise([{ bucket, key, versionId }], '') - .then(() => s3.send(new ListObjectVersionsCommand({ Bucket: bucket }))) - .then(res => res.Versions?.forEach(object => { - const params = [ - { - bucket, - key: object.Key, - versionId: object.VersionId, - }, - ]; - changeLockPromise(params, ''); - })) - .then(() => { - process.stdout.write('Emptying and deleting buckets\n'); - return bucketUtil.empty(bucket); - }) - .then(() => s3.send(new DeleteBucketCommand({ Bucket: bucket }))) - .catch(err => { - process.stdout.write('Error in afterEach'); - throw err; - })); + afterEach(() => + changeLockPromise([{ bucket, key, versionId }], '') + .then(() => s3.send(new ListObjectVersionsCommand({ Bucket: bucket }))) + .then(res => + res.Versions?.forEach(object => { + const params = [ + { + bucket, + key: object.Key, + versionId: object.VersionId, + }, + ]; + changeLockPromise(params, ''); + }), + ) + .then(() => { + process.stdout.write('Emptying and deleting buckets\n'); + return bucketUtil.empty(bucket); + }) + .then(() => s3.send(new DeleteBucketCommand({ Bucket: bucket }))) + .catch(err => { + process.stdout.write('Error in afterEach'); + throw err; + }), + ); it('should return object lock headers if set on the object', done => { - s3.send(new GetObjectCommand({ Bucket: bucket, Key: key })).then(res => { - assert.strictEqual(res.ObjectLockMode, mockMode); - const responseDate - = formatDate(res.ObjectLockRetainUntilDate); - const expectedDate = formatDate(mockDate); - assert.strictEqual(responseDate, expectedDate); - assert.strictEqual(res.ObjectLockLegalHoldStatus, 'ON'); - const objectWithLock = [ - { - bucket, - key, - versionId: res.VersionId, - }, - ]; - changeObjectLock(objectWithLock, '', done); - }).catch(done); + s3.send(new GetObjectCommand({ Bucket: bucket, Key: key })) + .then(res => { + assert.strictEqual(res.ObjectLockMode, mockMode); + const responseDate = formatDate(res.ObjectLockRetainUntilDate); + const expectedDate = formatDate(mockDate); + assert.strictEqual(responseDate, expectedDate); + assert.strictEqual(res.ObjectLockLegalHoldStatus, 'ON'); + const objectWithLock = [ + { + bucket, + key, + versionId: res.VersionId, + }, + ]; + changeObjectLock(objectWithLock, '', done); + }) + .catch(done); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/object/getMPU_compatibleHeaders.js b/tests/functional/aws-node-sdk/test/object/getMPU_compatibleHeaders.js index 22d1636c0d..31cf3efb78 100644 --- a/tests/functional/aws-node-sdk/test/object/getMPU_compatibleHeaders.js +++ b/tests/functional/aws-node-sdk/test/object/getMPU_compatibleHeaders.js @@ -13,114 +13,122 @@ const BucketUtility = require('../../lib/utility/bucket-util'); const bucketName = 'testgetmpubucket'; const objectName = 'key'; -describe('GET multipart upload object [Cache-Control, Content-Disposition, ' + -'Content-Encoding, Expires headers]', () => { - withV4(sigCfg => { - let bucketUtil; - let s3; - let uploadId; - const cacheControl = 'max-age=86400'; - const contentDisposition = 'attachment; filename="fname.ext";'; - const contentEncoding = 'aws-chunked,gzip'; - // AWS Node SDK requires Date object, ISO-8601 string, or - // a UNIX timestamp for Expires header - const expires = new Date(); +describe( + 'GET multipart upload object [Cache-Control, Content-Disposition, ' + 'Content-Encoding, Expires headers]', + () => { + withV4(sigCfg => { + let bucketUtil; + let s3; + let uploadId; + const cacheControl = 'max-age=86400'; + const contentDisposition = 'attachment; filename="fname.ext";'; + const contentEncoding = 'aws-chunked,gzip'; + // AWS Node SDK requires Date object, ISO-8601 string, or + // a UNIX timestamp for Expires header + const expires = new Date(); - before(() => { - const params = { - Bucket: bucketName, - Key: objectName, - CacheControl: cacheControl, - ContentDisposition: contentDisposition, - ContentEncoding: contentEncoding, - Expires: expires, - }; - bucketUtil = new BucketUtility('default', sigCfg); - s3 = bucketUtil.s3; - return bucketUtil.empty(bucketName) - .then(() => { - process.stdout.write('deleting bucket, just in case\n'); - return bucketUtil.deleteOne(bucketName); - }) - .catch(err => { - if (err.name !== 'NoSuchBucket') { - process.stdout.write(`${err}\n`); - throw err; - } - }) - .then(() => { - process.stdout.write('creating bucket\n'); - return s3.send(new CreateBucketCommand({ Bucket: bucketName })); - }) - .then(() => { - process.stdout.write('initiating multipart upload\n'); - return s3.send(new CreateMultipartUploadCommand(params)); - }) - .then(res => { - uploadId = res.UploadId; - return uploadId; - }) - .catch(err => { - process.stdout.write(`Error in before: ${err}\n`); - throw err; - }); - }); - after(() => { - process.stdout.write('Emptying bucket\n'); - return bucketUtil.empty(bucketName) - .then(() => { - process.stdout.write('Deleting bucket\n'); - return bucketUtil.deleteOne(bucketName); - }) - .catch(err => { - process.stdout.write('Error in after\n'); - throw err; - }); - }); - it('should return additional headers when get request is performed ' + - 'on MPU, when they are specified in creation of MPU', - () => { - const params = { Bucket: bucketName, Key: 'key', PartNumber: 1, - UploadId: uploadId }; - return s3.send(new UploadPartCommand(params)) - .catch(err => { - process.stdout.write(`Error in uploadPart ${err}\n`); - throw err; - }) - .then(res => { - process.stdout.write('about to complete multipart upload\n'); - return s3.send(new CompleteMultipartUploadCommand({ + before(() => { + const params = { Bucket: bucketName, Key: objectName, - UploadId: uploadId, - MultipartUpload: { - Parts: [ - { ETag: res.ETag, PartNumber: 1 }, - ], - }, - })); - }) - .catch(err => { - process.stdout.write(`Error completing upload ${err}\n`); - throw err; - }) - .then(() => { - process.stdout.write('about to get object\n'); - return s3.send(new GetObjectCommand({ - Bucket: bucketName, Key: objectName, - })); - }) - .catch(err => { - process.stdout.write(`Error getting object ${err}\n`); - throw err; - }) - .then(res => { - assert.strictEqual(res.CacheControl, cacheControl); - assert.strictEqual(res.ContentDisposition, contentDisposition); - assert.strictEqual(res.ContentEncoding, 'gzip'); - assert.strictEqual(res.Expires.toGMTString(), - expires.toGMTString()); + CacheControl: cacheControl, + ContentDisposition: contentDisposition, + ContentEncoding: contentEncoding, + Expires: expires, + }; + bucketUtil = new BucketUtility('default', sigCfg); + s3 = bucketUtil.s3; + return bucketUtil + .empty(bucketName) + .then(() => { + process.stdout.write('deleting bucket, just in case\n'); + return bucketUtil.deleteOne(bucketName); + }) + .catch(err => { + if (err.name !== 'NoSuchBucket') { + process.stdout.write(`${err}\n`); + throw err; + } + }) + .then(() => { + process.stdout.write('creating bucket\n'); + return s3.send(new CreateBucketCommand({ Bucket: bucketName })); + }) + .then(() => { + process.stdout.write('initiating multipart upload\n'); + return s3.send(new CreateMultipartUploadCommand(params)); + }) + .then(res => { + uploadId = res.UploadId; + return uploadId; + }) + .catch(err => { + process.stdout.write(`Error in before: ${err}\n`); + throw err; + }); + }); + after(() => { + process.stdout.write('Emptying bucket\n'); + return bucketUtil + .empty(bucketName) + .then(() => { + process.stdout.write('Deleting bucket\n'); + return bucketUtil.deleteOne(bucketName); + }) + .catch(err => { + process.stdout.write('Error in after\n'); + throw err; + }); }); + it( + 'should return additional headers when get request is performed ' + + 'on MPU, when they are specified in creation of MPU', + () => { + const params = { Bucket: bucketName, Key: 'key', PartNumber: 1, UploadId: uploadId }; + return s3 + .send(new UploadPartCommand(params)) + .catch(err => { + process.stdout.write(`Error in uploadPart ${err}\n`); + throw err; + }) + .then(res => { + process.stdout.write('about to complete multipart upload\n'); + return s3.send( + new CompleteMultipartUploadCommand({ + Bucket: bucketName, + Key: objectName, + UploadId: uploadId, + MultipartUpload: { + Parts: [{ ETag: res.ETag, PartNumber: 1 }], + }, + }), + ); + }) + .catch(err => { + process.stdout.write(`Error completing upload ${err}\n`); + throw err; + }) + .then(() => { + process.stdout.write('about to get object\n'); + return s3.send( + new GetObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); + }) + .catch(err => { + process.stdout.write(`Error getting object ${err}\n`); + throw err; + }) + .then(res => { + assert.strictEqual(res.CacheControl, cacheControl); + assert.strictEqual(res.ContentDisposition, contentDisposition); + assert.strictEqual(res.ContentEncoding, 'gzip'); + assert.strictEqual(res.Expires.toGMTString(), expires.toGMTString()); + }); + }, + ); }); - }); -}); + }, +); diff --git a/tests/functional/aws-node-sdk/test/object/getObjTagging.js b/tests/functional/aws-node-sdk/test/object/getObjTagging.js index 282866e7db..1bfbfc20c9 100644 --- a/tests/functional/aws-node-sdk/test/object/getObjTagging.js +++ b/tests/functional/aws-node-sdk/test/object/getObjTagging.js @@ -16,16 +16,18 @@ const bucketName = 'testtaggingbucket'; const objectName = 'testtaggingobject'; const objectNameAcl = 'testtaggingobjectacl'; -const taggingConfig = { TagSet: [ - { - Key: 'key1', - Value: 'value1', - }, - { - Key: 'key2', - Value: 'value2', - }, -] }; +const taggingConfig = { + TagSet: [ + { + Key: 'key1', + Value: 'value1', + }, + { + Key: 'key2', + Value: 'value2', + }, + ], +}; describe('GET object taggings', () => { withV4(sigCfg => { @@ -41,140 +43,174 @@ describe('GET object taggings', () => { afterEach(() => { process.stdout.write('Emptying bucket'); - return bucketUtil.empty(bucketName) - .then(() => { - process.stdout.write('Deleting bucket'); - return bucketUtil.deleteOne(bucketName); - }) - .catch(err => { - process.stdout.write('Error in afterEach'); - throw err; - }); + return bucketUtil + .empty(bucketName) + .then(() => { + process.stdout.write('Deleting bucket'); + return bucketUtil.deleteOne(bucketName); + }) + .catch(err => { + process.stdout.write('Error in afterEach'); + throw err; + }); }); it('should return appropriate tags after putting tags', async () => { - await s3.send(new PutObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - Tagging: taggingConfig, - })); + await s3.send( + new PutObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + Tagging: taggingConfig, + }), + ); - const data = await s3.send(new GetObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - })); + const data = await s3.send( + new GetObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); assert.deepStrictEqual(data.TagSet, taggingConfig.TagSet); }); it('should return no tag after putting and deleting tags', async () => { - await s3.send(new PutObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - Tagging: taggingConfig, - })); - await s3.send(new DeleteObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - })); - const data = await s3.send(new GetObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - })); + await s3.send( + new PutObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + Tagging: taggingConfig, + }), + ); + await s3.send( + new DeleteObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); + const data = await s3.send( + new GetObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); assert.deepStrictEqual(data.TagSet, []); }); - it('should return empty array after putting no tag', - async () => { - const data = await s3.send(new GetObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - })); + it('should return empty array after putting no tag', async () => { + const data = await s3.send( + new GetObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); assert.deepStrictEqual(data.TagSet, []); }); - it('should return NoSuchKey getting tag set to a non-existing object', - async () => { + it('should return NoSuchKey getting tag set to a non-existing object', async () => { try { - await s3.send(new GetObjectTaggingCommand({ - Bucket: bucketName, - Key: 'nonexisting', - })); + await s3.send( + new GetObjectTaggingCommand({ + Bucket: bucketName, + Key: 'nonexisting', + }), + ); throw new Error('Expected NoSuchKey error'); } catch (err) { checkError(err, 'NoSuchKey', 404); } }); - it('should return 403 AccessDenied getting tag set with another ' + - 'account', async () => { + it('should return 403 AccessDenied getting tag set with another ' + 'account', async () => { try { - await otherAccountS3.send(new GetObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - })); + await otherAccountS3.send( + new GetObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); throw new Error('Expected AccessDenied error'); } catch (err) { checkError(err, 'AccessDenied', 403); } }); - it('should return 403 AccessDenied getting tag with a different ' + - 'account to an object with ACL "public-read-write"', - async () => { - try { - await s3.send(new PutBucketAclCommand({ + it( + 'should return 403 AccessDenied getting tag with a different ' + + 'account to an object with ACL "public-read-write"', + async () => { + try { + await s3.send( + new PutBucketAclCommand({ + Bucket: bucketName, + ACL: 'public-read-write', + }), + ); + await otherAccountS3.send( + new GetObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); + throw new Error('Expected AccessDenied error'); + } catch (err) { + checkError(err, 'AccessDenied', 403); + } + }, + ); + + it( + 'should return 403 AccessDenied getting tag set to an object' + + ' in a bucket created with a different account', + async () => { + await s3.send( + new PutBucketAclCommand({ + Bucket: bucketName, + ACL: 'public-read-write', + }), + ); + await otherAccountS3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectNameAcl, + }), + ); + + try { + await otherAccountS3.send( + new GetObjectTaggingCommand({ + Bucket: bucketName, + Key: objectNameAcl, + }), + ); + throw new Error('Expected AccessDenied error'); + } catch (err) { + checkError(err, 'AccessDenied', 403); + } + }, + ); + + it('should get tag to an object in a bucket created with same ' + 'account', async () => { + await s3.send( + new PutBucketAclCommand({ Bucket: bucketName, ACL: 'public-read-write', - })); - await otherAccountS3.send(new GetObjectTaggingCommand({ + }), + ); + await otherAccountS3.send( + new PutObjectCommand({ Bucket: bucketName, - Key: objectName, - })); - throw new Error('Expected AccessDenied error'); - } catch (err) { - checkError(err, 'AccessDenied', 403); - } - }); - - it('should return 403 AccessDenied getting tag set to an object' + - ' in a bucket created with a different account', async () => { - await s3.send(new PutBucketAclCommand({ - Bucket: bucketName, - ACL: 'public-read-write', - })); - await otherAccountS3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectNameAcl, - })); + Key: objectNameAcl, + }), + ); - try { - await otherAccountS3.send(new GetObjectTaggingCommand({ + const data = await s3.send( + new GetObjectTaggingCommand({ Bucket: bucketName, Key: objectNameAcl, - })); - throw new Error('Expected AccessDenied error'); - } catch (err) { - checkError(err, 'AccessDenied', 403); - } - }); - - it('should get tag to an object in a bucket created with same ' + - 'account', async () => { - await s3.send(new PutBucketAclCommand({ - Bucket: bucketName, - ACL: 'public-read-write', - })); - await otherAccountS3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectNameAcl, - })); - - const data = await s3.send(new GetObjectTaggingCommand({ - Bucket: bucketName, - Key: objectNameAcl, - })); + }), + ); assert.deepStrictEqual(data.TagSet, []); }); diff --git a/tests/functional/aws-node-sdk/test/object/getObjectLegalHold.js b/tests/functional/aws-node-sdk/test/object/getObjectLegalHold.js index 1424897ced..b40d9551c9 100644 --- a/tests/functional/aws-node-sdk/test/object/getObjectLegalHold.js +++ b/tests/functional/aws-node-sdk/test/object/getObjectLegalHold.js @@ -20,7 +20,6 @@ const unlockedBucket = 'mock-bucket-no-lock'; const key = 'mock-object-legalhold'; const keyNoHold = 'mock-object-no-legalhold'; - describe('GET object legal hold', () => { withV4(sigCfg => { const bucketUtil = new BucketUtility('default', sigCfg); @@ -32,10 +31,12 @@ describe('GET object legal hold', () => { beforeEach(async () => { process.stdout.write('Putting buckets and objects\n'); process.stdout.write('Putting object legal hold\n'); - await s3.send(new CreateBucketCommand({ - Bucket: bucket, - ObjectLockEnabledForBucket: true, - })); + await s3.send( + new CreateBucketCommand({ + Bucket: bucket, + ObjectLockEnabledForBucket: true, + }), + ); await s3.send(new CreateBucketCommand({ Bucket: unlockedBucket })); await s3.send(new PutObjectCommand({ Bucket: unlockedBucket, Key: key })); await s3.send(new PutObjectCommand({ Bucket: bucket, Key: keyNoHold })); @@ -43,99 +44,141 @@ describe('GET object legal hold', () => { const res = await s3.send(new PutObjectCommand({ Bucket: bucket, Key: key })); versionId = res.VersionId; process.stdout.write('Putting object legal hold\n'); - await s3.send(new PutObjectLegalHoldCommand({ - Bucket: bucket, - Key: key, - LegalHold: { Status: 'ON' }, - })); + await s3.send( + new PutObjectLegalHoldCommand({ + Bucket: bucket, + Key: key, + LegalHold: { Status: 'ON' }, + }), + ); }); afterEach(() => { process.stdout.write('Removing object lock\n'); return changeLockPromise([{ bucket, key, versionId }], {}) - .then(() => { - process.stdout.write('Emptying and deleting buckets\n'); - return bucketUtil.empty(bucket); - }) - .then(() => bucketUtil.empty(unlockedBucket)) - .then(() => bucketUtil.deleteMany([bucket, unlockedBucket])) - .catch(err => { - process.stdout.write('Error in afterEach'); - throw err; - }); + .then(() => { + process.stdout.write('Emptying and deleting buckets\n'); + return bucketUtil.empty(bucket); + }) + .then(() => bucketUtil.empty(unlockedBucket)) + .then(() => bucketUtil.deleteMany([bucket, unlockedBucket])) + .catch(err => { + process.stdout.write('Error in afterEach'); + throw err; + }); }); - it('should return AccessDenied getting legal hold with another account', - () => otherAccountS3.send(new GetObjectLegalHoldCommand({ - Bucket: bucket, - Key: key, - })).then(() => { - throw new Error('Expected AccessDenied error'); - }).catch(err => { - checkError(err, 'AccessDenied', 403); - }) - ); + it('should return AccessDenied getting legal hold with another account', () => + otherAccountS3 + .send( + new GetObjectLegalHoldCommand({ + Bucket: bucket, + Key: key, + }), + ) + .then(() => { + throw new Error('Expected AccessDenied error'); + }) + .catch(err => { + checkError(err, 'AccessDenied', 403); + })); - it('should return MethodNotAllowed if object version is delete marker', () => s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: key, - })).then(res => s3.send(new GetObjectLegalHoldCommand({ - Bucket: bucket, - Key: key, - VersionId: res.VersionId, - })).then(() => { - throw new Error('Expected NoSuchKey error'); - }).catch(err => { - checkError(err, 'MethodNotAllowed', 405); - })).catch(err => { - assert.ifError(err); - }) - ); + it('should return MethodNotAllowed if object version is delete marker', () => + s3 + .send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: key, + }), + ) + .then(res => + s3 + .send( + new GetObjectLegalHoldCommand({ + Bucket: bucket, + Key: key, + VersionId: res.VersionId, + }), + ) + .then(() => { + throw new Error('Expected NoSuchKey error'); + }) + .catch(err => { + checkError(err, 'MethodNotAllowed', 405); + }), + ) + .catch(err => { + assert.ifError(err); + })); - it('should return NoSuchKey if latest version is delete marker', () => s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: key, - })).then(() => s3.send(new GetObjectLegalHoldCommand({ - Bucket: bucket, - Key: key, - })).then(() => { - throw new Error('Expected NoSuchKey error'); - }).catch(err => { - checkError(err, 'NoSuchKey', 404); - }) - ).catch(err => { - assert.ifError(err); - }) - ); + it('should return NoSuchKey if latest version is delete marker', () => + s3 + .send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: key, + }), + ) + .then(() => + s3 + .send( + new GetObjectLegalHoldCommand({ + Bucket: bucket, + Key: key, + }), + ) + .then(() => { + throw new Error('Expected NoSuchKey error'); + }) + .catch(err => { + checkError(err, 'NoSuchKey', 404); + }), + ) + .catch(err => { + assert.ifError(err); + })); - it('should return InvalidRequest error getting legal hold of object ' + - 'inside object lock disabled bucket', () => s3.send(new GetObjectLegalHoldCommand({ - Bucket: unlockedBucket, - Key: key, - })).then(() => { - throw new Error('Expected InvalidRequest error'); - }).catch(err => { - checkError(err, 'InvalidRequest', 400); - }) + it( + 'should return InvalidRequest error getting legal hold of object ' + 'inside object lock disabled bucket', + () => + s3 + .send( + new GetObjectLegalHoldCommand({ + Bucket: unlockedBucket, + Key: key, + }), + ) + .then(() => { + throw new Error('Expected InvalidRequest error'); + }) + .catch(err => { + checkError(err, 'InvalidRequest', 400); + }), ); - it('should return NoSuchObjectLockConfiguration if no legal hold set', () => - s3.send(new GetObjectLegalHoldCommand({ - Bucket: bucket, - Key: keyNoHold, - })).then(() => { - throw new Error('Expected NoSuchObjectLockConfiguration error'); - }).catch(err => { - checkError(err, 'NoSuchObjectLockConfiguration', 404); - }) - ); + it('should return NoSuchObjectLockConfiguration if no legal hold set', () => + s3 + .send( + new GetObjectLegalHoldCommand({ + Bucket: bucket, + Key: keyNoHold, + }), + ) + .then(() => { + throw new Error('Expected NoSuchObjectLockConfiguration error'); + }) + .catch(err => { + checkError(err, 'NoSuchObjectLockConfiguration', 404); + })); + + it('should get object legal hold', async () => { + const res = await s3.send( + new GetObjectLegalHoldCommand({ + Bucket: bucket, + Key: key, + }), + ); - it('should get object legal hold', async () => { - const res = await s3.send(new GetObjectLegalHoldCommand({ - Bucket: bucket, - Key: key, - })); - assert.deepStrictEqual(res.LegalHold, { Status: 'ON' }); await changeLockPromise([{ bucket, key, versionId }], {}); }); diff --git a/tests/functional/aws-node-sdk/test/object/getPartSize.js b/tests/functional/aws-node-sdk/test/object/getPartSize.js index fe9cc4924e..135fda36a2 100644 --- a/tests/functional/aws-node-sdk/test/object/getPartSize.js +++ b/tests/functional/aws-node-sdk/test/object/getPartSize.js @@ -30,8 +30,7 @@ const invalidPartNumbers = [-1, 0, maximumAllowedPartCount + 1]; let ETags = []; function checkNoError(err) { - assert.equal(err, null, - `Expected success, got error ${JSON.stringify(err)}`); + assert.equal(err, null, `Expected success, got error ${JSON.stringify(err)}`); } function generateContent(partNumber) { @@ -45,25 +44,31 @@ describe('Part size tests with object head', () => { let uploadId; function headObject(fields, cb) { - s3.send(new HeadObjectCommand({ - Bucket: bucket, - Key: object, - ...fields, - })).then(data => { - cb(null, data); - }).catch(err => { - cb(err); - }); + s3.send( + new HeadObjectCommand({ + Bucket: bucket, + Key: object, + ...fields, + }), + ) + .then(data => { + cb(null, data); + }) + .catch(err => { + cb(err); + }); } before(async () => { bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; await s3.send(new CreateBucketCommand({ Bucket: bucket })); - const uploadResult = await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: object - })); + const uploadResult = await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: object, + }), + ); uploadId = uploadResult.UploadId; const uploadPromises = partNumbers.map(async partNumber => { const uploadPartParams = { @@ -77,16 +82,20 @@ describe('Part size tests with object head', () => { return result.ETag; }); ETags = await Promise.all(uploadPromises); - await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: emptyObject, - Body: '', - })); - await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: nonMpuObject, - Body: generateContent(0), - })); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: emptyObject, + Body: '', + }), + ); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: nonMpuObject, + Body: generateContent(0), + }), + ); const completeParams = { Bucket: bucket, Key: object, @@ -102,37 +111,42 @@ describe('Part size tests with object head', () => { }); after(async () => { - await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: object - })); - await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: emptyObject - })); - await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: nonMpuObject - })); - await s3.send(new DeleteBucketCommand({ - Bucket: bucket - })); + await s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: object, + }), + ); + await s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: emptyObject, + }), + ); + await s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: nonMpuObject, + }), + ); + await s3.send( + new DeleteBucketCommand({ + Bucket: bucket, + }), + ); }); - it('should return the total size of the object ' + - 'when --part-number is not used', done => { - const totalSize = partNumbers.reduce((total, current) => - total + (bodySize + current + 1), 0); - headObject({}, (err, data) => { - checkNoError(err); - assert.equal(totalSize, data.ContentLength); - done(); - }); + it('should return the total size of the object ' + 'when --part-number is not used', done => { + const totalSize = partNumbers.reduce((total, current) => total + (bodySize + current + 1), 0); + headObject({}, (err, data) => { + checkNoError(err); + assert.equal(totalSize, data.ContentLength); + done(); }); + }); partNumbers.forEach(part => { - it(`should return the size of part ${part + 1} ` + - `when --part-number is set to ${part + 1}`, done => { + it(`should return the size of part ${part + 1} ` + `when --part-number is set to ${part + 1}`, done => { const partNumber = Number.parseInt(part, 10) + 1; const partSize = bodySize + partNumber; headObject({ PartNumber: partNumber }, (err, data) => { @@ -144,8 +158,7 @@ describe('Part size tests with object head', () => { }); invalidPartNumbers.forEach(part => { - it(`should return an error when --part-number is set to ${part}`, - done => { + it(`should return an error when --part-number is set to ${part}`, done => { headObject({ PartNumber: part }, err => { assert.equal(err.$metadata.httpStatusCode, 400); done(); @@ -154,8 +167,7 @@ describe('Part size tests with object head', () => { }); it('should return an error when incorrect --part-number is used', done => { - headObject({ PartNumber: partNumbers.length + 1 }, - err => { + headObject({ PartNumber: partNumbers.length + 1 }, err => { checkError(err, '', 416); done(); }); @@ -174,7 +186,7 @@ describe('Part size tests with object head', () => { checkError(err, '', 416); done(); }); - }); + }); it('should return content-length requesting part 1 of non-MPU object', done => { headObject({ Key: nonMpuObject, PartNumber: 1 }, (err, data) => { diff --git a/tests/functional/aws-node-sdk/test/object/getRange.js b/tests/functional/aws-node-sdk/test/object/getRange.js index 236d0e7618..44efc3ee7f 100644 --- a/tests/functional/aws-node-sdk/test/object/getRange.js +++ b/tests/functional/aws-node-sdk/test/object/getRange.js @@ -1,9 +1,5 @@ const assert = require('assert'); -const { - GetObjectCommand, - CreateBucketCommand, - PutObjectCommand -} = require('@aws-sdk/client-s3'); +const { GetObjectCommand, CreateBucketCommand, PutObjectCommand } = require('@aws-sdk/client-s3'); const withV4 = require('../support/withV4'); const BucketUtility = require('../../lib/utility/bucket-util'); @@ -44,12 +40,13 @@ describe('aws-node-sdk range test of large end position', () => { bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; await s3.send(new CreateBucketCommand({ Bucket: bucketName })); - await s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objName, - Body: Buffer.allocUnsafe(2890).fill(0, 0, 2800) - .fill(1, 2800), - })); + await s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objName, + Body: Buffer.allocUnsafe(2890).fill(0, 0, 2800).fill(1, 2800), + }), + ); }); afterEach(async () => { @@ -59,15 +56,13 @@ describe('aws-node-sdk range test of large end position', () => { await bucketUtil.deleteOne(bucketName); }); - it('should get the final 90 bytes of a 2890 byte object for a byte ' + - 'range of 2800-', - done => endRangeTest('bytes=2800-', 'bytes 2800-2889/2890', done) + it('should get the final 90 bytes of a 2890 byte object for a byte ' + 'range of 2800-', done => + endRangeTest('bytes=2800-', 'bytes 2800-2889/2890', done), ); - it('should get the final 90 bytes of a 2890 byte object for a byte ' + - 'range of 2800-Number.MAX_SAFE_INTEGER', - done => endRangeTest(`bytes=2800-${Number.MAX_SAFE_INTEGER}`, - 'bytes 2800-2889/2890', done) + it( + 'should get the final 90 bytes of a 2890 byte object for a byte ' + 'range of 2800-Number.MAX_SAFE_INTEGER', + done => endRangeTest(`bytes=2800-${Number.MAX_SAFE_INTEGER}`, 'bytes 2800-2889/2890', done), ); }); }); diff --git a/tests/functional/aws-node-sdk/test/object/getRetention.js b/tests/functional/aws-node-sdk/test/object/getRetention.js index 3c97a0fbaa..682f409246 100644 --- a/tests/functional/aws-node-sdk/test/object/getRetention.js +++ b/tests/functional/aws-node-sdk/test/object/getRetention.js @@ -32,7 +32,6 @@ const expectedConfig = { RetainUntilDate: new Date(retainDate), }; - describe('GET object retention', () => { withV4(sigCfg => { const bucketUtil = new BucketUtility('default', sigCfg); @@ -43,23 +42,27 @@ describe('GET object retention', () => { beforeEach(async () => { process.stdout.write('Putting buckets and objects\n'); - await s3.send(new CreateBucketCommand({ - Bucket: bucketName, - ObjectLockEnabledForBucket: true, - })); + await s3.send( + new CreateBucketCommand({ + Bucket: bucketName, + ObjectLockEnabledForBucket: true, + }), + ); await s3.send(new CreateBucketCommand({ Bucket: unlockedBucket })); await s3.send(new PutObjectCommand({ Bucket: unlockedBucket, Key: objectName })); await s3.send(new PutObjectCommand({ Bucket: bucketName, Key: noRetentionObject })); - + const res = await s3.send(new PutObjectCommand({ Bucket: bucketName, Key: objectName })); versionId = res.VersionId; - + process.stdout.write('Putting object retention\n'); - await s3.send(new PutObjectRetentionCommand({ - Bucket: bucketName, - Key: objectName, - Retention: retentionConfig, - })); + await s3.send( + new PutObjectRetentionCommand({ + Bucket: bucketName, + Key: objectName, + Retention: retentionConfig, + }), + ); }); afterEach(async () => { @@ -69,13 +72,14 @@ describe('GET object retention', () => { await bucketUtil.deleteMany([bucketName, unlockedBucket]); }); - it('should return AccessDenied putting retention with another account', - async () => { + it('should return AccessDenied putting retention with another account', async () => { try { - await otherAccountS3.send(new GetObjectRetentionCommand({ - Bucket: bucketName, - Key: objectName, - })); + await otherAccountS3.send( + new GetObjectRetentionCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); throw new Error('Expected AccessDenied error'); } catch (err) { checkError(err, 'AccessDenied', 403); @@ -84,10 +88,12 @@ describe('GET object retention', () => { it('should return NoSuchKey error if key does not exist', async () => { try { - await s3.send(new GetObjectRetentionCommand({ - Bucket: bucketName, - Key: 'thiskeydoesnotexist', - })); + await s3.send( + new GetObjectRetentionCommand({ + Bucket: bucketName, + Key: 'thiskeydoesnotexist', + }), + ); throw new Error('Expected NoSuchKey error'); } catch (err) { checkError(err, 'NoSuchKey', 404); @@ -96,52 +102,60 @@ describe('GET object retention', () => { it('should return NoSuchVersion error if version does not exist', async () => { try { - await s3.send(new GetObjectRetentionCommand({ - Bucket: bucketName, - Key: objectName, - VersionId: '012345678901234567890123456789012', - })); + await s3.send( + new GetObjectRetentionCommand({ + Bucket: bucketName, + Key: objectName, + VersionId: '012345678901234567890123456789012', + }), + ); throw new Error('Expected NoSuchVersion error'); } catch (err) { checkError(err, 'NoSuchVersion', 404); } }); - it('should return MethodNotAllowed if object version is delete marker', - async () => { + it('should return MethodNotAllowed if object version is delete marker', async () => { const res = await s3.send(new DeleteObjectCommand({ Bucket: bucketName, Key: objectName })); try { - await s3.send(new GetObjectRetentionCommand({ - Bucket: bucketName, - Key: objectName, - VersionId: res.VersionId, - })); + await s3.send( + new GetObjectRetentionCommand({ + Bucket: bucketName, + Key: objectName, + VersionId: res.VersionId, + }), + ); throw new Error('Expected MethodNotAllowed error'); } catch (err) { checkError(err, 'MethodNotAllowed', 405); } }); - it('should return InvalidRequest error getting retention to object ' + - 'in bucket with no object lock enabled', async () => { + it( + 'should return InvalidRequest error getting retention to object ' + 'in bucket with no object lock enabled', + async () => { + try { + await s3.send( + new GetObjectRetentionCommand({ + Bucket: unlockedBucket, + Key: objectName, + }), + ); + throw new Error('Expected InvalidRequest error'); + } catch (err) { + checkError(err, 'InvalidRequest', 400); + } + }, + ); + + it('should return NoSuchObjectLockConfiguration if no retention set', async () => { try { - await s3.send(new GetObjectRetentionCommand({ - Bucket: unlockedBucket, - Key: objectName, - })); - throw new Error('Expected InvalidRequest error'); - } catch (err) { - checkError(err, 'InvalidRequest', 400); - } - }); - - it('should return NoSuchObjectLockConfiguration if no retention set', - async () => { - try { - await s3.send(new GetObjectRetentionCommand({ - Bucket: bucketName, - Key: noRetentionObject, - })); + await s3.send( + new GetObjectRetentionCommand({ + Bucket: bucketName, + Key: noRetentionObject, + }), + ); throw new Error('Expected NoSuchObjectLockConfiguration error'); } catch (err) { checkError(err, 'NoSuchObjectLockConfiguration', 404); @@ -149,13 +163,14 @@ describe('GET object retention', () => { }); it('should get object retention', async () => { - const res = await s3.send(new GetObjectRetentionCommand({ - Bucket: bucketName, - Key: objectName, - })); + const res = await s3.send( + new GetObjectRetentionCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); assert.deepStrictEqual(res.Retention, expectedConfig); - await changeLockPromise([ - { bucket: bucketName, key: objectName, versionId }], ''); + await changeLockPromise([{ bucket: bucketName, key: objectName, versionId }], ''); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/object/initiateMPU.js b/tests/functional/aws-node-sdk/test/object/initiateMPU.js index eb6416973a..97fb816895 100644 --- a/tests/functional/aws-node-sdk/test/object/initiateMPU.js +++ b/tests/functional/aws-node-sdk/test/object/initiateMPU.js @@ -8,8 +8,7 @@ const { const withV4 = require('../support/withV4'); const BucketUtility = require('../../lib/utility/bucket-util'); -const genMaxSizeMetaHeaders - = require('../../lib/utility/genMaxSizeMetaHeaders'); +const genMaxSizeMetaHeaders = require('../../lib/utility/genMaxSizeMetaHeaders'); const { generateMultipleTagQuery } = require('../../lib/utility/tagging'); const bucket = `initiatempubucket${Date.now()}`; @@ -28,59 +27,69 @@ describe('Initiate MPU', () => { afterEach(async () => await bucketUtil.deleteOne(bucket)); - it('should return InvalidRedirectLocation if initiate MPU ' + - 'with x-amz-website-redirect-location header that does not start ' + - 'with \'http://\', \'https://\' or \'/\'', async () => { - const params = { - Bucket: bucket, - Key: key, - WebsiteRedirectLocation: 'google.com' - }; - - try { - await s3.send(new CreateMultipartUploadCommand(params)); - throw new Error('Expected InvalidRedirectLocation error'); - } catch (err) { - assert.strictEqual(err.name, 'InvalidRedirectLocation'); - assert.strictEqual(err.$metadata.httpStatusCode, 400); - } - }); + it( + 'should return InvalidRedirectLocation if initiate MPU ' + + 'with x-amz-website-redirect-location header that does not start ' + + "with 'http://', 'https://' or '/'", + async () => { + const params = { + Bucket: bucket, + Key: key, + WebsiteRedirectLocation: 'google.com', + }; - it('should return InvalidStorageClass error when x-amz-storage-class header is provided ' + - 'and not equal to STANDARD', done => { - s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: key, - StorageClass: 'COLD', - })).then(() => { - throw new Error('Expected InvalidStorageClass error'); - }).catch(err => { - assert.strictEqual(err.name, 'InvalidStorageClass'); - assert.strictEqual(err.$metadata.httpStatusCode, 400); - done(); - }); - }); + try { + await s3.send(new CreateMultipartUploadCommand(params)); + throw new Error('Expected InvalidRedirectLocation error'); + } catch (err) { + assert.strictEqual(err.name, 'InvalidRedirectLocation'); + assert.strictEqual(err.$metadata.httpStatusCode, 400); + } + }, + ); + + it( + 'should return InvalidStorageClass error when x-amz-storage-class header is provided ' + + 'and not equal to STANDARD', + done => { + s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + StorageClass: 'COLD', + }), + ) + .then(() => { + throw new Error('Expected InvalidStorageClass error'); + }) + .catch(err => { + assert.strictEqual(err.name, 'InvalidStorageClass'); + assert.strictEqual(err.$metadata.httpStatusCode, 400); + done(); + }); + }, + ); it('should return KeyTooLong error when key is longer than 915 bytes', done => { - s3.send(new CreateMultipartUploadCommand({ Bucket: bucket, Key: 'a'.repeat(916) })) - .catch(err => { + s3.send(new CreateMultipartUploadCommand({ Bucket: bucket, Key: 'a'.repeat(916) })).catch(err => { assert.strictEqual(err.name, 'KeyTooLong'); assert.strictEqual(err.$metadata.httpStatusCode, 400); done(); }); }); - it('should return error if initiating MPU w/ > 2KB user-defined md', - async () => { + it('should return error if initiating MPU w/ > 2KB user-defined md', async () => { const metadata = genMaxSizeMetaHeaders(); const params = { Bucket: bucket, Key: key, Metadata: metadata }; const data = await s3.send(new CreateMultipartUploadCommand(params)); const uploadId = data.UploadId; - await s3.send(new AbortMultipartUploadCommand({ - Bucket: bucket, - Key: key, - UploadId: uploadId, - })); + await s3.send( + new AbortMultipartUploadCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + }), + ); metadata.header0 = `${metadata.header0}${'0'}`; try { await s3.send(new CreateMultipartUploadCommand(params)); @@ -91,57 +100,62 @@ describe('Initiate MPU', () => { } }); - it('should return error if initiating MPU w/ > 2KB user-defined md', - async () => { - const metadata = genMaxSizeMetaHeaders(); - const params = { Bucket: bucket, Key: key, Metadata: metadata }; - const data = await s3.send(new CreateMultipartUploadCommand(params)); - const uploadId = data.UploadId; - await s3.send(new AbortMultipartUploadCommand({ + it('should return error if initiating MPU w/ > 2KB user-defined md', async () => { + const metadata = genMaxSizeMetaHeaders(); + const params = { Bucket: bucket, Key: key, Metadata: metadata }; + const data = await s3.send(new CreateMultipartUploadCommand(params)); + const uploadId = data.UploadId; + await s3.send( + new AbortMultipartUploadCommand({ Bucket: bucket, Key: key, UploadId: uploadId, - })); - metadata.header0 = `${metadata.header0}${'0'}`; - try { - await s3.send(new CreateMultipartUploadCommand(params)); - throw new Error('Expected MetadataTooLarge error'); - } catch (err) { - assert.strictEqual(err.name, 'MetadataTooLarge'); - assert.strictEqual(err.$metadata.httpStatusCode, 400); - } + }), + ); + metadata.header0 = `${metadata.header0}${'0'}`; + try { + await s3.send(new CreateMultipartUploadCommand(params)); + throw new Error('Expected MetadataTooLarge error'); + } catch (err) { + assert.strictEqual(err.name, 'MetadataTooLarge'); + assert.strictEqual(err.$metadata.httpStatusCode, 400); + } }); describe('with tag set', () => { - it('should be able to put object with 10 tags', - async () => { + it('should be able to put object with 10 tags', async () => { const taggingConfig = generateMultipleTagQuery(10); - await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: key, - Tagging: taggingConfig, - })); + await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + Tagging: taggingConfig, + }), + ); }); it('should allow putting 50 tags', async () => { const taggingConfig = generateMultipleTagQuery(50); - await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: key, - Tagging: taggingConfig, - })); + await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + Tagging: taggingConfig, + }), + ); }); - it('should return BadRequest if putting more that 50 tags', - async () => { + it('should return BadRequest if putting more that 50 tags', async () => { const taggingConfig = generateMultipleTagQuery(51); - + try { - await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: key, - Tagging: taggingConfig, - })); + await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + Tagging: taggingConfig, + }), + ); throw new Error('Expected BadRequest error'); } catch (err) { assert.strictEqual(err.name, 'BadRequest'); @@ -149,16 +163,17 @@ describe('Initiate MPU', () => { } }); - it('should return InvalidArgument creating mpu tag with ' + - 'invalid characters: %', async () => { + it('should return InvalidArgument creating mpu tag with ' + 'invalid characters: %', async () => { const value = 'value1%'; - + try { - await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: key, - Tagging: `key1=${value}`, - })); + await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + Tagging: `key1=${value}`, + }), + ); throw new Error('Expected InvalidArgument error'); } catch (err) { assert.strictEqual(err.name, 'InvalidArgument'); @@ -166,14 +181,15 @@ describe('Initiate MPU', () => { } }); - it('should return InvalidArgument creating mpu with ' + - 'bad encoded tags', async () => { + it('should return InvalidArgument creating mpu with ' + 'bad encoded tags', async () => { try { - await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: key, - Tagging: 'key1==value1', - })); + await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + Tagging: 'key1==value1', + }), + ); throw new Error('Expected InvalidArgument error'); } catch (err) { assert.strictEqual(err.name, 'InvalidArgument'); @@ -183,11 +199,13 @@ describe('Initiate MPU', () => { it('should return InvalidArgument if tag with no key', async () => { try { - await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: key, - Tagging: '=value1', - })); + await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + Tagging: '=value1', + }), + ); throw new Error('Expected InvalidArgument error'); } catch (err) { assert.strictEqual(err.name, 'InvalidArgument'); @@ -195,14 +213,15 @@ describe('Initiate MPU', () => { } }); - it('should return InvalidArgument if using the same key twice', - async () => { + it('should return InvalidArgument if using the same key twice', async () => { try { - await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: key, - Tagging: 'key1=value1&key1=value2', - })); + await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + Tagging: 'key1=value1&key1=value2', + }), + ); throw new Error('Expected InvalidArgument error'); } catch (err) { assert.strictEqual(err.name, 'InvalidArgument'); @@ -210,14 +229,15 @@ describe('Initiate MPU', () => { } }); - it('should return InvalidArgument if using the same key twice ' + - 'and empty tags', async () => { + it('should return InvalidArgument if using the same key twice ' + 'and empty tags', async () => { try { - await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Tagging: '&&&&&&&&&&&&&&&&&key1=value1&key1=value2', - })); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Tagging: '&&&&&&&&&&&&&&&&&key1=value1&key1=value2', + }), + ); throw new Error('Expected InvalidArgument error'); } catch (err) { assert.strictEqual(err.name, 'InvalidArgument'); diff --git a/tests/functional/aws-node-sdk/test/object/listParts.js b/tests/functional/aws-node-sdk/test/object/listParts.js index f670b33eb8..d02cb93fb2 100644 --- a/tests/functional/aws-node-sdk/test/object/listParts.js +++ b/tests/functional/aws-node-sdk/test/object/listParts.js @@ -24,35 +24,59 @@ describe('List parts', () => { beforeEach(async () => { await s3.send(new CreateBucketCommand({ Bucket: bucket })); - const res = await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, Key: key })); + const res = await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + }), + ); uploadId = res.UploadId; - await s3.send(new UploadPartCommand({ Bucket: bucket, Key: key, - PartNumber: 1, UploadId: uploadId, Body: bodyFirstPart, - })); - const secondRes = await s3.send(new UploadPartCommand({ - Bucket: bucket, Key: key, - PartNumber: 2, UploadId: uploadId, Body: bodySecondPart, - })); + await s3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: key, + PartNumber: 1, + UploadId: uploadId, + Body: bodyFirstPart, + }), + ); + const secondRes = await s3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: key, + PartNumber: 2, + UploadId: uploadId, + Body: bodySecondPart, + }), + ); secondEtag = secondRes.ETag; }); afterEach(async () => { process.stdout.write('Emptying bucket'); - await s3.send(new AbortMultipartUploadCommand({ - Bucket: bucket, Key: key, UploadId: uploadId, - })); + await s3.send( + new AbortMultipartUploadCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + }), + ); await bucketUtil.empty(bucket); process.stdout.write('Deleting bucket'); await bucketUtil.deleteOne(bucket); }); - it('should only list the second part', () => s3.send(new ListPartsCommand({ - Bucket: bucket, - Key: key, - PartNumberMarker: '1', - UploadId: uploadId, - })).then(data => { + it('should only list the second part', () => + s3 + .send( + new ListPartsCommand({ + Bucket: bucket, + Key: key, + PartNumberMarker: '1', + UploadId: uploadId, + }), + ) + .then(data => { assert.strictEqual(data.Parts[0].PartNumber, 2); assert.strictEqual(data.Parts[0].Size, 20); assert.strictEqual(`${data.Parts[0].ETag}`, secondEtag); @@ -64,36 +88,59 @@ describe('List parts', () => { function createPart(sigCfg, bucketUtil, s3, key) { let uploadId; - return s3.send(new CreateBucketCommand({ Bucket: bucket })) - .then(() => s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, Key: key }))) - .then(res => { - uploadId = res.UploadId; - return s3.send(new UploadPartCommand({ Bucket: bucket, Key: key, - PartNumber: 1, UploadId: uploadId, Body: bodyFirstPart })); - }) - .then(() => Promise.resolve(uploadId)); + return s3 + .send(new CreateBucketCommand({ Bucket: bucket })) + .then(() => + s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + }), + ), + ) + .then(res => { + uploadId = res.UploadId; + return s3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: key, + PartNumber: 1, + UploadId: uploadId, + Body: bodyFirstPart, + }), + ); + }) + .then(() => Promise.resolve(uploadId)); } function deletePart(s3, bucketUtil, key, uploadId) { process.stdout.write('Emptying bucket'); - return s3.send(new AbortMultipartUploadCommand({ - Bucket: bucket, Key: key, UploadId: uploadId, - })) - .then(() => bucketUtil.empty(bucket)) - .then(() => { - process.stdout.write('Deleting bucket'); - return bucketUtil.deleteOne(bucket); - }); + return s3 + .send( + new AbortMultipartUploadCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + }), + ) + .then(() => bucketUtil.empty(bucket)) + .then(() => { + process.stdout.write('Deleting bucket'); + return bucketUtil.deleteOne(bucket); + }); } function testFunc(s3, bucket, key, uploadId) { - return s3.send(new ListPartsCommand({ - Bucket: bucket, - Key: key, - UploadId: uploadId, - })).then(data => { + return s3 + .send( + new ListPartsCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + }), + ) + .then(data => { assert.strictEqual(data.Key, key); }); } @@ -106,17 +153,15 @@ describe('List parts - object keys with special characters: `&`', () => { const key = '&'; beforeEach(() => - createPart(sigCfg, bucketUtil, s3, key) - .then(res => { + createPart(sigCfg, bucketUtil, s3, key).then(res => { uploadId = res; return Promise.resolve(); - }) + }), ); afterEach(() => deletePart(s3, bucketUtil, key, uploadId)); - it('should list parts of an object with `&` in its key', - () => testFunc(s3, bucket, key, uploadId)); + it('should list parts of an object with `&` in its key', () => testFunc(s3, bucket, key, uploadId)); }); }); @@ -128,39 +173,35 @@ describe('List parts - object keys with special characters: `"`', () => { const key = '"quot'; beforeEach(() => - createPart(sigCfg, bucketUtil, s3, key) - .then(res => { + createPart(sigCfg, bucketUtil, s3, key).then(res => { uploadId = res; return Promise.resolve(); - }) + }), ); afterEach(() => deletePart(s3, bucketUtil, key, uploadId)); - it('should list parts of an object with `"` in its key', - () => testFunc(s3, bucket, key, uploadId)); + it('should list parts of an object with `"` in its key', () => testFunc(s3, bucket, key, uploadId)); }); }); -describe('List parts - object keys with special characters: `\'`', () => { +describe("List parts - object keys with special characters: `'`", () => { withV4(sigCfg => { const bucketUtil = new BucketUtility('default', sigCfg); const s3 = bucketUtil.s3; let uploadId; - const key = '\'apos'; + const key = "'apos"; beforeEach(() => - createPart(sigCfg, bucketUtil, s3, key) - .then(res => { + createPart(sigCfg, bucketUtil, s3, key).then(res => { uploadId = res; return Promise.resolve(); - }) + }), ); afterEach(() => deletePart(s3, bucketUtil, key, uploadId)); - it('should list parts of an object with `\'` in its key', - () => testFunc(s3, bucket, key, uploadId)); + it("should list parts of an object with `'` in its key", () => testFunc(s3, bucket, key, uploadId)); }); }); @@ -172,17 +213,15 @@ describe('List parts - object keys with special characters: `<`', () => { const key = ' - createPart(sigCfg, bucketUtil, s3, key) - .then(res => { + createPart(sigCfg, bucketUtil, s3, key).then(res => { uploadId = res; return Promise.resolve(); - }) + }), ); afterEach(() => deletePart(s3, bucketUtil, key, uploadId)); - it('should list parts of an object with `<` in its key', - () => testFunc(s3, bucket, key, uploadId)); + it('should list parts of an object with `<` in its key', () => testFunc(s3, bucket, key, uploadId)); }); }); @@ -194,16 +233,14 @@ describe('List parts - object keys with special characters: `>`', () => { const key = '>gt'; beforeEach(() => - createPart(sigCfg, bucketUtil, s3, key) - .then(res => { + createPart(sigCfg, bucketUtil, s3, key).then(res => { uploadId = res; return Promise.resolve(); - }) + }), ); afterEach(() => deletePart(s3, bucketUtil, key, uploadId)); - it('should list parts of an object with `>` in its key', - () => testFunc(s3, bucket, key, uploadId)); + it('should list parts of an object with `>` in its key', () => testFunc(s3, bucket, key, uploadId)); }); }); diff --git a/tests/functional/aws-node-sdk/test/object/mpu.js b/tests/functional/aws-node-sdk/test/object/mpu.js index 58eaebd8d3..4bf8f1b8a2 100644 --- a/tests/functional/aws-node-sdk/test/object/mpu.js +++ b/tests/functional/aws-node-sdk/test/object/mpu.js @@ -1,4 +1,3 @@ - const assert = require('assert'); const { CreateBucketCommand, @@ -42,22 +41,22 @@ function getExpectedObj(res, data) { NextUploadIdMarker: uploadId, MaxUploads: maxUploads, IsTruncated: false, - Uploads: [{ - UploadId: uploadId, - Key: objectKey, - Initiated: initiated, - StorageClass: 'STANDARD', - Owner: - { - DisplayName: displayName, - ID: userId, - }, - Initiator: + Uploads: [ { - DisplayName: displayName, - ID: userId, + UploadId: uploadId, + Key: objectKey, + Initiated: initiated, + StorageClass: 'STANDARD', + Owner: { + DisplayName: displayName, + ID: userId, + }, + Initiator: { + DisplayName: displayName, + ID: userId, + }, }, - }], + ], }; // If no `prefixVal` is given, it should not be included in the response. @@ -93,20 +92,24 @@ describe('aws-node-sdk test suite of listMultipartUploads', () => // The owner of the bucket will also be the MPU upload owner. data.displayName = ownerRes.DisplayName; data.userId = ownerRes.ID; - - const mpuRes = await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: objectKey, - })); - data.uploadId = mpuRes.UploadId; + + const mpuRes = await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: objectKey, + }), + ); + data.uploadId = mpuRes.UploadId; }); afterEach(async () => { - await s3.send(new AbortMultipartUploadCommand({ - Bucket: bucket, - Key: objectKey, - UploadId: data.uploadId, - })); + await s3.send( + new AbortMultipartUploadCommand({ + Bucket: bucket, + Key: objectKey, + UploadId: data.uploadId, + }), + ); await bucketUtil.empty(bucket); await bucketUtil.deleteOne(bucket); }); @@ -122,23 +125,26 @@ describe('aws-node-sdk test suite of listMultipartUploads', () => data.delimiter = 'test-delimiter'; data.maxUploads = 1; // eslint-disable-next-line no-unused-vars - const {$metadata, ...res } = await s3.send(new ListMultipartUploadsCommand({ - Bucket: bucket, - Prefix: 'to', - Delimiter: 'test-delimiter', - MaxUploads: 1, - })); + const { $metadata, ...res } = await s3.send( + new ListMultipartUploadsCommand({ + Bucket: bucket, + Prefix: 'to', + Delimiter: 'test-delimiter', + MaxUploads: 1, + }), + ); checkValues(res, data); }); it('should list 0 multipart uploads when MaxUploads is 0', async () => { data.maxUploads = 0; // eslint-disable-next-line no-unused-vars - const { $metadata , ...res } = await s3.send(new ListMultipartUploadsCommand({ - Bucket: bucket, - MaxUploads: 0, - })); + const { $metadata, ...res } = await s3.send( + new ListMultipartUploadsCommand({ + Bucket: bucket, + MaxUploads: 0, + }), + ); checkValues(res, data); }); - }) -); + })); diff --git a/tests/functional/aws-node-sdk/test/object/mpuOrder.js b/tests/functional/aws-node-sdk/test/object/mpuOrder.js index 7ef4ef3d66..fea9648b74 100644 --- a/tests/functional/aws-node-sdk/test/object/mpuOrder.js +++ b/tests/functional/aws-node-sdk/test/object/mpuOrder.js @@ -23,13 +23,13 @@ function checkError(err, statusCode, code) { const body = Buffer.alloc(1024 * 1024 * 5, 'a'); const testsOrder = [ - { values: [3, 8, 1000], err: false }, - { values: [8, 3, 1000], err: true }, - { values: [8, 1000, 3], err: true }, - { values: [1000, 3, 8], err: true }, - { values: [3, 1000, 8], err: true }, - { values: [1000, 8, 3], err: true }, - { values: [3, 3, 1000], err: true }, + { values: [3, 8, 1000], err: false }, + { values: [8, 3, 1000], err: true }, + { values: [8, 1000, 3], err: true }, + { values: [1000, 3, 8], err: true }, + { values: [3, 1000, 8], err: true }, + { values: [1000, 8, 3], err: true }, + { values: [3, 3, 1000], err: true }, ]; describe('More MPU tests', () => { @@ -41,33 +41,41 @@ describe('More MPU tests', () => { bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; await s3.send(new CreateBucketCommand({ Bucket: bucket })); - const mpuRes = await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: object - })); + const mpuRes = await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: object, + }), + ); this.currentTest.UploadId = mpuRes.UploadId; - const part1000Res = await s3.send(new UploadPartCommand({ - Bucket: bucket, - Key: object, - PartNumber: 1000, - Body: body, - UploadId: this.currentTest.UploadId - })); + const part1000Res = await s3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: object, + PartNumber: 1000, + Body: body, + UploadId: this.currentTest.UploadId, + }), + ); this.currentTest.Etag = part1000Res.ETag; - await s3.send(new UploadPartCommand({ - Bucket: bucket, - Key: object, - PartNumber: 3, - Body: body, - UploadId: this.currentTest.UploadId - })); - await s3.send(new UploadPartCommand({ - Bucket: bucket, - Key: object, - PartNumber: 8, - Body: body, - UploadId: this.currentTest.UploadId - })); + await s3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: object, + PartNumber: 3, + Body: body, + UploadId: this.currentTest.UploadId, + }), + ); + await s3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: object, + PartNumber: 8, + Body: body, + UploadId: this.currentTest.UploadId, + }), + ); }); afterEach(async () => { @@ -76,47 +84,53 @@ describe('More MPU tests', () => { }); testsOrder.forEach(testOrder => { - it('should complete MPU by concatenating the parts in ' + - `the following order: ${testOrder.values}`, async function itF() { - try { - await s3.send(new CompleteMultipartUploadCommand({ - Bucket: bucket, - Key: object, - MultipartUpload: { - Parts: [ - { - ETag: this.test.Etag, - PartNumber: testOrder.values[0], + it( + 'should complete MPU by concatenating the parts in ' + `the following order: ${testOrder.values}`, + async function itF() { + try { + await s3.send( + new CompleteMultipartUploadCommand({ + Bucket: bucket, + Key: object, + MultipartUpload: { + Parts: [ + { + ETag: this.test.Etag, + PartNumber: testOrder.values[0], + }, + { + ETag: this.test.Etag, + PartNumber: testOrder.values[1], + }, + { + ETag: this.test.Etag, + PartNumber: testOrder.values[2], + }, + ], }, - { - ETag: this.test.Etag, - PartNumber: testOrder.values[1], - }, - { - ETag: this.test.Etag, - PartNumber: testOrder.values[2], - }, - ], - }, - UploadId: this.test.UploadId - })); - - if (testOrder.err) { - throw new Error('Expected InvalidPartOrder error but operation succeeded'); - } - } catch (err) { - if (testOrder.err) { - checkError(err, 400, 'InvalidPartOrder'); - await s3.send(new AbortMultipartUploadCommand({ - Bucket: bucket, - Key: object, - UploadId: this.test.UploadId, - })); - } else { - throw err; + UploadId: this.test.UploadId, + }), + ); + + if (testOrder.err) { + throw new Error('Expected InvalidPartOrder error but operation succeeded'); + } + } catch (err) { + if (testOrder.err) { + checkError(err, 400, 'InvalidPartOrder'); + await s3.send( + new AbortMultipartUploadCommand({ + Bucket: bucket, + Key: object, + UploadId: this.test.UploadId, + }), + ); + } else { + throw err; + } } - } - }); + }, + ); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/object/multiObjectDelete.js b/tests/functional/aws-node-sdk/test/object/multiObjectDelete.js index 7aa6d9babf..4d591b0ff2 100644 --- a/tests/functional/aws-node-sdk/test/object/multiObjectDelete.js +++ b/tests/functional/aws-node-sdk/test/object/multiObjectDelete.js @@ -23,8 +23,7 @@ const bucketName = 'multi-object-delete-234-634'; const key = 'key'; function checkNoError(err) { - assert.equal(err, null, - `Expected success, got error ${JSON.stringify(err)}`); + assert.equal(err, null, `Expected success, got error ${JSON.stringify(err)}`); } function sortList(list) { @@ -32,7 +31,7 @@ function sortList(list) { // Handle both string arrays and object arrays const keyA = typeof a === 'string' ? a : a.Key; const keyB = typeof b === 'string' ? b : b.Key; - + // Extract numeric part from keys like 'key1', 'key2', 'key10', etc. const getNumber = key => parseInt(key.replace(/^key/, ''), 10); const numA = getNumber(keyA); @@ -43,7 +42,7 @@ function sortList(list) { function createObjectsList(size, versionIds) { const objects = []; - for (let i = 1; i < (size + 1); i++) { + for (let i = 1; i < size + 1; i++) { objects.push({ Key: `${key}${i}`, }); @@ -80,11 +79,13 @@ describe('Multi-Object Delete Success', function success() { await Promise.race(queued); queued.splice(0, queued.findIndex(p => p === queued[0]) + 1); } - const result = s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: key, - Body: 'somebody', - })); + const result = s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: key, + Body: 'somebody', + }), + ); queued.push(result); return result; }; @@ -103,39 +104,48 @@ describe('Multi-Object Delete Success', function success() { it('should batch delete 1000 objects', done => { const objects = createObjectsList(1000); - s3.send(new DeleteObjectsCommand({ - Bucket: bucketName, - Delete: { - Objects: objects, - Quiet: false, - }, - })).then(res => { - if (this.httpResponse?.body?.toString() - .indexOf(' obj.Key)), sortList(objects.map(obj => obj.Key))); - return done(); - }).catch(err => done(err)); + s3.send( + new DeleteObjectsCommand({ + Bucket: bucketName, + Delete: { + Objects: objects, + Quiet: false, + }, + }), + ) + .then(res => { + if (this.httpResponse?.body?.toString().indexOf(' obj.Key)), + sortList(objects.map(obj => obj.Key)), + ); + return done(); + }) + .catch(err => done(err)); }); it('should batch delete 1000 objects quietly', done => { const objects = createObjectsList(1000); - s3.send(new DeleteObjectsCommand({ - Bucket: bucketName, - Delete: { - Objects: objects, - Quiet: true, - }, - })).then(res => { - if (this.httpResponse?.body?.toString() - .indexOf(' done(err)); + s3.send( + new DeleteObjectsCommand({ + Bucket: bucketName, + Delete: { + Objects: objects, + Quiet: true, + }, + }), + ) + .then(res => { + if (this.httpResponse?.body?.toString().indexOf(' done(err)); }); }); @@ -147,8 +157,7 @@ describe('Multi-Object Delete Error Responses', () => { beforeEach(() => { bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; - return s3.send(new CreateBucketCommand({ Bucket: bucketName })) - .catch(err => { + return s3.send(new CreateBucketCommand({ Bucket: bucketName })).catch(err => { process.stdout.write(`Error creating bucket: ${err}\n`); throw err; }); @@ -159,57 +168,71 @@ describe('Multi-Object Delete Error Responses', () => { await s3.send(new DeleteBucketCommand({ Bucket: bucketName })); }); - it('should return error if request deletion of more than 1000 objects', - () => { - const objects = createObjectsList(1001); - return s3.send(new DeleteObjectsCommand({ - Bucket: bucketName, - Delete: { - Objects: objects, - }, - })).catch(err => { + it('should return error if request deletion of more than 1000 objects', () => { + const objects = createObjectsList(1001); + return s3 + .send( + new DeleteObjectsCommand({ + Bucket: bucketName, + Delete: { + Objects: objects, + }, + }), + ) + .catch(err => { checkError(err, 'MalformedXML', 400); }); - }); + }); - it('should return error if request deletion of 0 objects', - () => { - const objects = createObjectsList(0); - return s3.send(new DeleteObjectsCommand({ - Bucket: bucketName, - Delete: { - Objects: objects, - }, - })).catch(err => { + it('should return error if request deletion of 0 objects', () => { + const objects = createObjectsList(0); + return s3 + .send( + new DeleteObjectsCommand({ + Bucket: bucketName, + Delete: { + Objects: objects, + }, + }), + ) + .catch(err => { checkError(err, 'MalformedXML', 400); }); - }); + }); - it('should return no error if try to delete non-existent objects', - () => { - const objects = createObjectsList(1000); - return s3.send(new DeleteObjectsCommand({ - Bucket: bucketName, - Delete: { - Objects: objects, - }, - })).then(res => { + it('should return no error if try to delete non-existent objects', () => { + const objects = createObjectsList(1000); + return s3 + .send( + new DeleteObjectsCommand({ + Bucket: bucketName, + Delete: { + Objects: objects, + }, + }), + ) + .then(res => { assert.strictEqual(res.Deleted.length, 1000); - }).catch(err => { + }) + .catch(err => { checkNoError(err); }); - }); + }); it('should return error if no such bucket', () => { const objects = createObjectsList(1); - return s3.send(new DeleteObjectsCommand({ - Bucket: 'nosuchbucket2323292093', - Delete: { - Objects: objects, - }, - })).catch(err => { - checkError(err, 'NoSuchBucket', 404); - }); + return s3 + .send( + new DeleteObjectsCommand({ + Bucket: 'nosuchbucket2323292093', + Delete: { + Objects: objects, + }, + }), + ) + .catch(err => { + checkError(err, 'NoSuchBucket', 404); + }); }); }); }); @@ -224,26 +247,30 @@ describe('Multi-Object Delete Access', function access() { signatureVersion: 'v4', }); s3 = bucketUtil.s3; - return s3.send(new CreateBucketCommand({ Bucket: bucketName })) - .catch(err => { - process.stdout.write(`Error creating bucket: ${err}\n`); - throw err; - }) - .then(() => { - const createObjects = []; - for (let i = 1; i < 501; i++) { - createObjects.push(s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: `${key}${i}`, - Body: 'somebody', - }))); - } - return Promise.all(createObjects) + return s3 + .send(new CreateBucketCommand({ Bucket: bucketName })) .catch(err => { - process.stdout.write(`Error creating objects: ${err}\n`); + process.stdout.write(`Error creating bucket: ${err}\n`); throw err; + }) + .then(() => { + const createObjects = []; + for (let i = 1; i < 501; i++) { + createObjects.push( + s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: `${key}${i}`, + Body: 'somebody', + }), + ), + ); + } + return Promise.all(createObjects).catch(err => { + process.stdout.write(`Error creating objects: ${err}\n`); + throw err; + }); }); - }); }); after(async () => { @@ -251,8 +278,7 @@ describe('Multi-Object Delete Access', function access() { await s3.send(new DeleteBucketCommand({ Bucket: bucketName })); }); - it('should return access denied error for each object where no acl ' + - 'permission', () => { + it('should return access denied error for each object where no acl ' + 'permission', () => { const objects = createObjectsList(500); const errorList = createObjectsList(500); errorList.forEach(obj => { @@ -260,38 +286,47 @@ describe('Multi-Object Delete Access', function access() { item.Code = 'AccessDenied'; item.Message = 'Access Denied'; }); - return otherAccountS3.send(new DeleteObjectsCommand({ - Bucket: bucketName, - Delete: { - Objects: objects, - Quiet: false, - }, - })).then(res => { - assert.strictEqual(res.Deleted, undefined); - assert.strictEqual(res.Errors.length, 500); - assert.deepStrictEqual(sortList(res.Errors), sortList(errorList)); - }).catch(err => { - checkNoError(err); - }); + return otherAccountS3 + .send( + new DeleteObjectsCommand({ + Bucket: bucketName, + Delete: { + Objects: objects, + Quiet: false, + }, + }), + ) + .then(res => { + assert.strictEqual(res.Deleted, undefined); + assert.strictEqual(res.Errors.length, 500); + assert.deepStrictEqual(sortList(res.Errors), sortList(errorList)); + }) + .catch(err => { + checkNoError(err); + }); }); it('should batch delete objects where requester has permission', () => { const objects = createObjectsList(500); - return s3.send(new DeleteObjectsCommand({ - Bucket: bucketName, - Delete: { - Objects: objects, - Quiet: false, - }, - })).then(res => { - assert.strictEqual(res.Deleted.length, 500); - }).catch(err => { - checkNoError(err); - }); + return s3 + .send( + new DeleteObjectsCommand({ + Bucket: bucketName, + Delete: { + Objects: objects, + Quiet: false, + }, + }), + ) + .then(res => { + assert.strictEqual(res.Deleted.length, 500); + }) + .catch(err => { + checkNoError(err); + }); }); }); - describe('Multi-Object Delete with Object Lock', () => { let bucketUtil; let s3; @@ -303,45 +338,56 @@ describe('Multi-Object Delete with Object Lock', () => { signatureVersion: 'v4', }); s3 = bucketUtil.s3; - return s3.send(new CreateBucketCommand({ - Bucket: bucketName, - ObjectLockEnabledForBucket: true, - })) - .then(() => s3.send(new PutObjectLockConfigurationCommand({ - Bucket: bucketName, - ObjectLockConfiguration: { - ObjectLockEnabled: 'Enabled', - Rule: { - DefaultRetention: { - Days: 1, - Mode: 'GOVERNANCE', - }, - }, - }, - }))) - .catch(err => { - process.stdout.write(`Error creating bucket: ${err}\n`); - throw err; - }) - .then(() => { - for (let i = 1; i < 6; i++) { - createObjects.push(s3.send(new PutObjectCommand({ + return s3 + .send( + new CreateBucketCommand({ Bucket: bucketName, - Key: `${key}${i}`, - Body: 'somebody', - }))); - } - return Promise.all(createObjects) - .then(res => { - res.forEach(r => { - versionIds.push(r.VersionId); - }); - }) + ObjectLockEnabledForBucket: true, + }), + ) + .then(() => + s3.send( + new PutObjectLockConfigurationCommand({ + Bucket: bucketName, + ObjectLockConfiguration: { + ObjectLockEnabled: 'Enabled', + Rule: { + DefaultRetention: { + Days: 1, + Mode: 'GOVERNANCE', + }, + }, + }, + }), + ), + ) .catch(err => { - process.stdout.write(`Error creating objects: ${err}\n`); + process.stdout.write(`Error creating bucket: ${err}\n`); throw err; + }) + .then(() => { + for (let i = 1; i < 6; i++) { + createObjects.push( + s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: `${key}${i}`, + Body: 'somebody', + }), + ), + ); + } + return Promise.all(createObjects) + .then(res => { + res.forEach(r => { + versionIds.push(r.VersionId); + }); + }) + .catch(err => { + process.stdout.write(`Error creating objects: ${err}\n`); + throw err; + }); }); - }); }); after(async () => { @@ -351,45 +397,61 @@ describe('Multi-Object Delete with Object Lock', () => { it('should not delete locked objects', () => { const objects = createObjectsList(5, versionIds); - return s3.send(new DeleteObjectsCommand({ - Bucket: bucketName, - Delete: { - Objects: objects, - Quiet: false, - }, - })).then(res => { - assert.strictEqual(res.Errors.length, 5); - res.Errors.forEach(err => assert.strictEqual(err.Code, 'AccessDenied')); - }); - }); - - it('should not delete locked objects with GOVERNANCE ' + - 'retention mode and bypass header when object is legal hold enabled', () => { - const objects = createObjectsList(5, versionIds); - const putObjectLegalHolds = []; - for (let i = 1; i < 6; i++) { - putObjectLegalHolds.push(s3.send(new PutObjectLegalHoldCommand({ - Bucket: bucketName, - Key: `${key}${i}`, - LegalHold: { - Status: 'ON', - }, - }))); - } - return Promise.all(putObjectLegalHolds) - .then(() => s3.send(new DeleteObjectsCommand({ - Bucket: bucketName, - Delete: { - Objects: objects, - Quiet: false, - }, - BypassGovernanceRetention: true, - }))).then(res => { + return s3 + .send( + new DeleteObjectsCommand({ + Bucket: bucketName, + Delete: { + Objects: objects, + Quiet: false, + }, + }), + ) + .then(res => { assert.strictEqual(res.Errors.length, 5); res.Errors.forEach(err => assert.strictEqual(err.Code, 'AccessDenied')); }); }); + it( + 'should not delete locked objects with GOVERNANCE ' + + 'retention mode and bypass header when object is legal hold enabled', + () => { + const objects = createObjectsList(5, versionIds); + const putObjectLegalHolds = []; + for (let i = 1; i < 6; i++) { + putObjectLegalHolds.push( + s3.send( + new PutObjectLegalHoldCommand({ + Bucket: bucketName, + Key: `${key}${i}`, + LegalHold: { + Status: 'ON', + }, + }), + ), + ); + } + return Promise.all(putObjectLegalHolds) + .then(() => + s3.send( + new DeleteObjectsCommand({ + Bucket: bucketName, + Delete: { + Objects: objects, + Quiet: false, + }, + BypassGovernanceRetention: true, + }), + ), + ) + .then(res => { + assert.strictEqual(res.Errors.length, 5); + res.Errors.forEach(err => assert.strictEqual(err.Code, 'AccessDenied')); + }); + }, + ); + it('should delete locked objects after retention period has expired', () => { const objects = createObjectsList(5, versionIds); const objectsCopy = JSON.parse(JSON.stringify(objects)); @@ -405,34 +467,44 @@ describe('Multi-Object Delete with Object Lock', () => { date: moment().subtract(10, 'days').toISOString(), }; return changeLockPromise(objectsCopy, newRetention) - .then(() => s3.send(new DeleteObjectsCommand({ - Bucket: bucketName, - Delete: { - Objects: objects, - Quiet: false, - }, - }))).then(res => { - assert.strictEqual(res.Deleted.length, 5); - }).catch(err => { - checkNoError(err); - }); + .then(() => + s3.send( + new DeleteObjectsCommand({ + Bucket: bucketName, + Delete: { + Objects: objects, + Quiet: false, + }, + }), + ), + ) + .then(res => { + assert.strictEqual(res.Deleted.length, 5); + }) + .catch(err => { + checkNoError(err); + }); }); - it('should delete locked objects with GOVERNANCE ' + - 'retention mode and bypass header', () => { + it('should delete locked objects with GOVERNANCE ' + 'retention mode and bypass header', () => { const objects = createObjectsList(5, versionIds); - return s3.send(new DeleteObjectsCommand({ - Bucket: bucketName, - Delete: { - Objects: objects, - Quiet: false, - }, - BypassGovernanceRetention: true, - })).then(res => { - assert.strictEqual(res.Deleted.length, 5); - assert.strictEqual(res.Errors, undefined); - }).catch(err => { - checkNoError(err); - }); + return s3 + .send( + new DeleteObjectsCommand({ + Bucket: bucketName, + Delete: { + Objects: objects, + Quiet: false, + }, + BypassGovernanceRetention: true, + }), + ) + .then(res => { + assert.strictEqual(res.Deleted.length, 5); + assert.strictEqual(res.Errors, undefined); + }) + .catch(err => { + checkNoError(err); + }); }); }); diff --git a/tests/functional/aws-node-sdk/test/object/objectCopy.js b/tests/functional/aws-node-sdk/test/object/objectCopy.js index c7b6ceedcc..e50fcdb5a5 100644 --- a/tests/functional/aws-node-sdk/test/object/objectCopy.js +++ b/tests/functional/aws-node-sdk/test/object/objectCopy.js @@ -11,12 +11,11 @@ const { GetObjectTaggingCommand, PutObjectCommand, GetObjectAclCommand, - PutObjectAclCommand + PutObjectAclCommand, } = require('@aws-sdk/client-s3'); const { taggingTests } = require('../../lib/utility/tagging'); -const genMaxSizeMetaHeaders - = require('../../lib/utility/genMaxSizeMetaHeaders'); +const genMaxSizeMetaHeaders = require('../../lib/utility/genMaxSizeMetaHeaders'); const constants = require('../../../../../constants'); const sourceBucketName = 'supersourcebucket8102016'; @@ -58,8 +57,7 @@ const otherAccountS3 = otherAccountBucketUtility.s3; const itSkipIfE2E = process.env.S3_END_TO_END ? it.skip : it; function checkNoError(err) { - assert.equal(err, null, - `Expected success, got error ${JSON.stringify(err)}`); + assert.equal(err, null, `Expected success, got error ${JSON.stringify(err)}`); } function dateFromNow(diff) { @@ -72,7 +70,6 @@ function dateConvert(d) { return new Date(d); } - describe('Object Copy', () => { withV4(sigCfg => { let bucketUtil; @@ -81,7 +78,6 @@ describe('Object Copy', () => { let etagTrim; let lastModified; - before(async () => { try { bucketUtil = new BucketUtility('default', sigCfg); @@ -99,26 +95,35 @@ describe('Object Copy', () => { await bucketUtil.createOne(destBucketName); }); - beforeEach(() => s3.send(new PutObjectCommand({ - Bucket: sourceBucketName, - Key: sourceObjName, - Body: content, - Metadata: originalMetadata, - CacheControl: originalCacheControl, - ContentDisposition: originalContentDisposition, - ContentEncoding: originalContentEncoding, - Expires: originalExpires, - Tagging: originalTagging, - })).then(res => { - etag = res.ETag; - etagTrim = etag.substring(1, etag.length - 1); - return s3.send(new HeadObjectCommand({ - Bucket: sourceBucketName, - Key: sourceObjName, - })); - }).then(res => { - lastModified = res.LastModified; - })); + beforeEach(() => + s3 + .send( + new PutObjectCommand({ + Bucket: sourceBucketName, + Key: sourceObjName, + Body: content, + Metadata: originalMetadata, + CacheControl: originalCacheControl, + ContentDisposition: originalContentDisposition, + ContentEncoding: originalContentEncoding, + Expires: originalExpires, + Tagging: originalTagging, + }), + ) + .then(res => { + etag = res.ETag; + etagTrim = etag.substring(1, etag.length - 1); + return s3.send( + new HeadObjectCommand({ + Bucket: sourceBucketName, + Key: sourceObjName, + }), + ); + }) + .then(res => { + lastModified = res.LastModified; + }), + ); afterEach(async () => { await bucketUtil.empty(sourceBucketName, true); @@ -128,24 +133,35 @@ describe('Object Copy', () => { after(async () => await bucketUtil.deleteMany([sourceBucketName, destBucketName])); function requestCopy(fields, cb) { - s3.send(new CopyObjectCommand(Object.assign({ - Bucket: destBucketName, - Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - }, fields))).then(res => { - cb(null, res); - }).catch(cb); + s3.send( + new CopyObjectCommand( + Object.assign( + { + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + }, + fields, + ), + ), + ) + .then(res => { + cb(null, res); + }) + .catch(cb); } async function successCopyCheck(error, response, copyVersionMetadata, destBucketName, destObjName) { checkNoError(error); assert.strictEqual(response.ETag, etag); const copyLastModified = new Date(response.LastModified).toGMTString(); - - const res = await s3.send(new GetObjectCommand({ - Bucket: destBucketName, - Key: destObjName - })); + + const res = await s3.send( + new GetObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + }), + ); assert.strictEqual(res.StorageClass, undefined); const bodyString = await res.Body.transformToString(); assert.strictEqual(bodyString, content); @@ -154,157 +170,248 @@ describe('Object Copy', () => { } function checkSuccessTagging(key, value, cb) { - s3.send(new GetObjectTaggingCommand({ Bucket: destBucketName, Key: destObjName })).then(data => { - assert.strictEqual(data.TagSet[0].Key, key); - assert.strictEqual(data.TagSet[0].Value, value); - cb(); - }).catch(err => { - checkNoError(err); - cb(err); - }); + s3.send(new GetObjectTaggingCommand({ Bucket: destBucketName, Key: destObjName })) + .then(data => { + assert.strictEqual(data.TagSet[0].Key, key); + assert.strictEqual(data.TagSet[0].Value, value); + cb(); + }) + .catch(err => { + checkNoError(err); + cb(err); + }); } function checkNoTagging(cb) { - s3.send(new GetObjectTaggingCommand({ Bucket: destBucketName, Key: destObjName })).then(data => { - assert.strictEqual(data.TagSet.length, 0); - cb(); - }).catch(err => { - checkNoError(err); - cb(err); - }); - } - - it('should copy an object from a source bucket to a different ' + - 'destination bucket and copy the metadata if no metadata directive ' + - 'header provided', async () => { - const res = await s3.send(new CopyObjectCommand({ - Bucket: destBucketName, - Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}` - })); - await successCopyCheck(null, res.CopyObjectResult, originalMetadata, - destBucketName, destObjName); - }); - - it('should copy an object from a source bucket to a different ' + - 'destination bucket and copy the tag set if no tagging directive' + - 'header provided', done => { - s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}` })).then(() => { - checkSuccessTagging(originalTagKey, originalTagValue, done); - }).catch(err => { + s3.send(new GetObjectTaggingCommand({ Bucket: destBucketName, Key: destObjName })) + .then(data => { + assert.strictEqual(data.TagSet.length, 0); + cb(); + }) + .catch(err => { checkNoError(err); + cb(err); }); - }); + } + + it( + 'should copy an object from a source bucket to a different ' + + 'destination bucket and copy the metadata if no metadata directive ' + + 'header provided', + async () => { + const res = await s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + }), + ); + await successCopyCheck(null, res.CopyObjectResult, originalMetadata, destBucketName, destObjName); + }, + ); + + it( + 'should copy an object from a source bucket to a different ' + + 'destination bucket and copy the tag set if no tagging directive' + + 'header provided', + done => { + s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + }), + ) + .then(() => { + checkSuccessTagging(originalTagKey, originalTagValue, done); + }) + .catch(err => { + checkNoError(err); + }); + }, + ); - it('should return 400 InvalidArgument if invalid tagging ' + - 'directive', done => { - s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - TaggingDirective: 'COCO' })).then(() => { + it('should return 400 InvalidArgument if invalid tagging ' + 'directive', done => { + s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + TaggingDirective: 'COCO', + }), + ) + .then(() => { done(new Error('Expected 400 InvalidArgument error')); - }).catch(err => { + }) + .catch(err => { checkError(err, 'InvalidArgument', 400); done(); }); }); it('should return 400 KeyTooLong if key is longer than 915 bytes', done => { - s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: 'a'.repeat(916), - CopySource: `${sourceBucketName}/${sourceObjName}` })).then(() => { + s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: 'a'.repeat(916), + CopySource: `${sourceBucketName}/${sourceObjName}`, + }), + ) + .then(() => { done(new Error('Expected 400 KeyTooLong error')); - }).catch(err => { + }) + .catch(err => { checkError(err, 'KeyTooLong', 400); done(); }); }); - it('should copy an object from a source bucket to a different ' + - 'destination bucket and copy the tag set if COPY tagging ' + - 'directive header provided', done => { - s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - TaggingDirective: 'COPY' })).then(() => { - checkSuccessTagging(originalTagKey, originalTagValue, done); - }).catch(err => { - checkNoError(err); - }); - }); + it( + 'should copy an object from a source bucket to a different ' + + 'destination bucket and copy the tag set if COPY tagging ' + + 'directive header provided', + done => { + s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + TaggingDirective: 'COPY', + }), + ) + .then(() => { + checkSuccessTagging(originalTagKey, originalTagValue, done); + }) + .catch(err => { + checkNoError(err); + }); + }, + ); - it('should copy an object and tag set if COPY ' + - 'included as tag directive header (and ignore any new ' + - 'tag set sent with copy request)', done => { - s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - TaggingDirective: 'COPY', - Tagging: newTagging, - })).then(() => { - s3.send(new GetObjectCommand({ Bucket: destBucketName, - Key: destObjName })).then(res => { - assert.deepStrictEqual(res.Metadata, originalMetadata); - done(); - }).catch(err => { - checkNoError(err); - done(err); - }); - }).catch(err => { - checkNoError(err); - }); - }); + it( + 'should copy an object and tag set if COPY ' + + 'included as tag directive header (and ignore any new ' + + 'tag set sent with copy request)', + done => { + s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + TaggingDirective: 'COPY', + Tagging: newTagging, + }), + ) + .then(() => { + s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })) + .then(res => { + assert.deepStrictEqual(res.Metadata, originalMetadata); + done(); + }) + .catch(err => { + checkNoError(err); + done(err); + }); + }) + .catch(err => { + checkNoError(err); + }); + }, + ); - it('should copy an object from a source to the same destination ' + - 'updating tag if REPLACE tagging directive header provided', - done => { - s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - TaggingDirective: 'REPLACE', Tagging: newTagging })).then(() => { - checkSuccessTagging(newTagKey, newTagValue, done); - }).catch(err => { - checkNoError(err); - done(err); - }); - }); + it( + 'should copy an object from a source to the same destination ' + + 'updating tag if REPLACE tagging directive header provided', + done => { + s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + TaggingDirective: 'REPLACE', + Tagging: newTagging, + }), + ) + .then(() => { + checkSuccessTagging(newTagKey, newTagValue, done); + }) + .catch(err => { + checkNoError(err); + done(err); + }); + }, + ); - it('should copy an object from a source to the same destination ' + - 'return no tag if REPLACE tagging directive header provided but ' + - '"x-amz-tagging" header is not specified', done => { - s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - TaggingDirective: 'REPLACE' })).then(() => { - checkNoTagging(done); - }).catch(err => { - checkNoError(err); - done(err); - }); - }); + it( + 'should copy an object from a source to the same destination ' + + 'return no tag if REPLACE tagging directive header provided but ' + + '"x-amz-tagging" header is not specified', + done => { + s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + TaggingDirective: 'REPLACE', + }), + ) + .then(() => { + checkNoTagging(done); + }) + .catch(err => { + checkNoError(err); + done(err); + }); + }, + ); - it('should copy an object from a source to the same destination ' + - 'return no tag if COPY tagging directive header but provided from ' + - 'an empty object', done => { - s3.send(new PutObjectCommand({ Bucket: sourceBucketName, Key: 'emptyobject' })).then(() => { - s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: `${sourceBucketName}/emptyobject`, - TaggingDirective: 'COPY' })).then(() => { - checkNoTagging(done); - }).catch(err => { - checkNoError(err); - done(err); + it( + 'should copy an object from a source to the same destination ' + + 'return no tag if COPY tagging directive header but provided from ' + + 'an empty object', + done => { + s3.send(new PutObjectCommand({ Bucket: sourceBucketName, Key: 'emptyobject' })).then(() => { + s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/emptyobject`, + TaggingDirective: 'COPY', + }), + ) + .then(() => { + checkNoTagging(done); + }) + .catch(err => { + checkNoError(err); + done(err); + }); }); - }); - }); + }, + ); - it('should copy an object from a source to the same destination ' + - 'updating tag if REPLACE tagging directive header provided', - done => { - s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - TaggingDirective: 'REPLACE', Tagging: newTagging })).then(() => { - checkSuccessTagging(newTagKey, newTagValue, done); - }).catch(err => { - checkNoError(err); - done(err); - }); - }); + it( + 'should copy an object from a source to the same destination ' + + 'updating tag if REPLACE tagging directive header provided', + done => { + s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + TaggingDirective: 'REPLACE', + Tagging: newTagging, + }), + ) + .then(() => { + checkSuccessTagging(newTagKey, newTagValue, done); + }) + .catch(err => { + checkNoError(err); + done(err); + }); + }, + ); describe('Copy object updating tag set', () => { taggingTests.forEach(taggingTest => { @@ -312,63 +419,77 @@ describe('Object Copy', () => { const key = encodeURIComponent(taggingTest.tag.key); const value = encodeURIComponent(taggingTest.tag.value); const tagging = `${key}=${value}`; - const params = { Bucket: destBucketName, Key: destObjName, + const params = { + Bucket: destBucketName, + Key: destObjName, CopySource: `${sourceBucketName}/${sourceObjName}`, - TaggingDirective: 'REPLACE', Tagging: tagging }; - s3.send(new CopyObjectCommand(params)).then(() => checkSuccessTagging(taggingTest.tag.key, - taggingTest.tag.value, done)).catch(err => { - if (taggingTest.error) { - checkError(err, taggingTest.error, taggingTest.code); - return done(); - } - assert.equal(err, null, 'Expected success, ' + - `got error ${JSON.stringify(err)}`); - return checkSuccessTagging(taggingTest.tag.key, - taggingTest.tag.value, done); - }); + TaggingDirective: 'REPLACE', + Tagging: tagging, + }; + s3.send(new CopyObjectCommand(params)) + .then(() => checkSuccessTagging(taggingTest.tag.key, taggingTest.tag.value, done)) + .catch(err => { + if (taggingTest.error) { + checkError(err, taggingTest.error, taggingTest.code); + return done(); + } + assert.equal(err, null, 'Expected success, ' + `got error ${JSON.stringify(err)}`); + return checkSuccessTagging(taggingTest.tag.key, taggingTest.tag.value, done); + }); }); }); }); - it('should also copy additional headers (CacheControl, ' + - 'ContentDisposition, ContentEncoding, Expires) when copying an ' + - 'object from a source bucket to a different destination bucket', done => { - s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}` })).then(() => { - s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })).then(res => { - assert.strictEqual(res.CacheControl, - originalCacheControl); - assert.strictEqual(res.ContentDisposition, - originalContentDisposition); - // Should remove V4 streaming value 'aws-chunked' - // to be compatible with AWS behavior - assert.strictEqual(res.ContentEncoding, - 'base64,' - ); - assert.strictEqual(res.Expires.toGMTString(), - originalExpires.toGMTString()); - done(); - }).catch(err => { - checkNoError(err); - done(err); - }); - }).catch(err => { + it( + 'should also copy additional headers (CacheControl, ' + + 'ContentDisposition, ContentEncoding, Expires) when copying an ' + + 'object from a source bucket to a different destination bucket', + done => { + s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + }), + ) + .then(() => { + s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })) + .then(res => { + assert.strictEqual(res.CacheControl, originalCacheControl); + assert.strictEqual(res.ContentDisposition, originalContentDisposition); + // Should remove V4 streaming value 'aws-chunked' + // to be compatible with AWS behavior + assert.strictEqual(res.ContentEncoding, 'base64,'); + assert.strictEqual(res.Expires.toGMTString(), originalExpires.toGMTString()); + done(); + }) + .catch(err => { + checkNoError(err); + done(err); + }); + }) + .catch(err => { checkNoError(err); done(err); }); - }); + }, + ); - it('should copy an object from a source bucket to a different ' + - 'key in the same bucket', async () => { - const res = await s3.send(new CopyObjectCommand({ Bucket: sourceBucketName, Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}` })); - await successCopyCheck(null, res.CopyObjectResult, originalMetadata, - sourceBucketName, destObjName); - }); + it('should copy an object from a source bucket to a different ' + 'key in the same bucket', async () => { + const res = await s3.send( + new CopyObjectCommand({ + Bucket: sourceBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + }), + ); + await successCopyCheck(null, res.CopyObjectResult, originalMetadata, sourceBucketName, destObjName); + }); // TODO: see S3C-3482, figure out why this test fails in Integration builds - itSkipIfE2E('should not return error if copying object w/ > ' + - '2KB user-defined md and COPY directive', done => { + itSkipIfE2E( + 'should not return error if copying object w/ > ' + '2KB user-defined md and COPY directive', + done => { const metadata = genMaxSizeMetaHeaders(); const params = { Bucket: destBucketName, @@ -377,198 +498,261 @@ describe('Object Copy', () => { MetadataDirective: 'COPY', Metadata: metadata, }; - s3.send(new CopyObjectCommand(params)).then(() => { - // add one more byte to be over the limit - metadata.header0 = `${metadata.header0}${'0'}`; - s3.send(new CopyObjectCommand(params)).then(() => { - done(); - }).catch(err => { + s3.send(new CopyObjectCommand(params)) + .then(() => { + // add one more byte to be over the limit + metadata.header0 = `${metadata.header0}${'0'}`; + s3.send(new CopyObjectCommand(params)) + .then(() => { + done(); + }) + .catch(err => { + assert.strictEqual(err, null, `Unexpected err: ${err}`); + done(err); + }); + }) + .catch(err => { assert.strictEqual(err, null, `Unexpected err: ${err}`); done(err); }); - }).catch(err => { - assert.strictEqual(err, null, `Unexpected err: ${err}`); - done(err); - }); - }); + }, + ); // TODO: see S3C-3482, figure out why this test fails in Integration builds - itSkipIfE2E('should return error if copying object w/ > 2KB ' + - 'user-defined md and REPLACE directive', async () => { + itSkipIfE2E( + 'should return error if copying object w/ > 2KB ' + 'user-defined md and REPLACE directive', + async () => { try { const metadata = genMaxSizeMetaHeaders(); const params = { Bucket: destBucketName, Key: destObjName, CopySource: `${sourceBucketName}/${sourceObjName}`, + MetadataDirective: 'REPLACE', + Metadata: metadata, + }; + await s3.send(new CopyObjectCommand(params)); + // add one more byte to be over the limit + metadata.header0 = `${metadata.header0}${'0'}`; + await s3.send(new CopyObjectCommand(params)); + assert.fail('Expected MetadataTooLarge error'); + } catch (err) { + assert.strictEqual(err.name, 'MetadataTooLarge'); + assert.strictEqual(err.$metadata.httpStatusCode, 400); + } + }, + ); + + it('should copy an object from a source to the same destination ' + '(update metadata)', async () => { + const res = await s3.send( + new CopyObjectCommand({ + Bucket: sourceBucketName, + Key: sourceObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, MetadataDirective: 'REPLACE', - Metadata: metadata, - }; - await s3.send(new CopyObjectCommand(params)); - // add one more byte to be over the limit - metadata.header0 = `${metadata.header0}${'0'}`; - await s3.send(new CopyObjectCommand(params)); - assert.fail('Expected MetadataTooLarge error'); - } catch (err) { - assert.strictEqual(err.name, 'MetadataTooLarge'); - assert.strictEqual(err.$metadata.httpStatusCode, 400); - } - }); - - it('should copy an object from a source to the same destination ' + - '(update metadata)', async () => { - const res = await s3.send(new CopyObjectCommand({ Bucket: sourceBucketName, Key: sourceObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - MetadataDirective: 'REPLACE', - Metadata: newMetadata })); - await successCopyCheck(null, res.CopyObjectResult, newMetadata, - sourceBucketName, sourceObjName); - }); - - it('should copy an object and replace the metadata if replace ' + - 'included as metadata directive header', async () => { - const res = await s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - MetadataDirective: 'REPLACE', - Metadata: newMetadata })); - await successCopyCheck(null, res.CopyObjectResult, newMetadata, - destBucketName, destObjName); - }); - - it('should copy an object and replace ContentType if replace ' + - 'included as a metadata directive header, and new ContentType is ' + - 'provided', async () => { - await s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - MetadataDirective: 'REPLACE', - ContentType: 'image', - })); - const res = await s3.send(new GetObjectCommand({ Bucket: destBucketName, - Key: destObjName })); - assert.strictEqual(res.ContentType, 'image'); + Metadata: newMetadata, + }), + ); + await successCopyCheck(null, res.CopyObjectResult, newMetadata, sourceBucketName, sourceObjName); }); - it('should copy an object and keep ContentType if replace ' + - 'included as a metadata directive header, but no new ContentType ' + - 'is provided', done => { - s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - MetadataDirective: 'REPLACE', - })).then(() => { - s3.send(new GetObjectCommand({ Bucket: destBucketName, - Key: destObjName })).then(res => { - assert.strictEqual(res.ContentType, 'application/octet-stream'); - done(); - }).catch(err => { - checkNoError(err); - done(err); + it( + 'should copy an object and replace the metadata if replace ' + 'included as metadata directive header', + async () => { + const res = await s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + MetadataDirective: 'REPLACE', + Metadata: newMetadata, + }), + ); + await successCopyCheck(null, res.CopyObjectResult, newMetadata, destBucketName, destObjName); + }, + ); + + it( + 'should copy an object and replace ContentType if replace ' + + 'included as a metadata directive header, and new ContentType is ' + + 'provided', + async () => { + await s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + MetadataDirective: 'REPLACE', + ContentType: 'image', + }), + ); + const res = await s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })); + assert.strictEqual(res.ContentType, 'image'); + }, + ); + + it( + 'should copy an object and keep ContentType if replace ' + + 'included as a metadata directive header, but no new ContentType ' + + 'is provided', + done => { + s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + MetadataDirective: 'REPLACE', + }), + ).then(() => { + s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })) + .then(res => { + assert.strictEqual(res.ContentType, 'application/octet-stream'); + done(); + }) + .catch(err => { + checkNoError(err); + done(err); + }); }); - }); - }); + }, + ); - it('should also replace additional headers if replace ' + - 'included as metadata directive header and new headers are ' + - 'specified', done => { - s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - MetadataDirective: 'REPLACE', - CacheControl: newCacheControl, - ContentDisposition: newContentDisposition, - ContentEncoding: newContentEncoding, - Expires: newExpires, - })).then(() => { - s3.send(new GetObjectCommand({ Bucket: destBucketName, - Key: destObjName })).then(res => { - assert.strictEqual(res.CacheControl, newCacheControl); - assert.strictEqual(res.ContentDisposition, - newContentDisposition); - // Should remove V4 streaming value 'aws-chunked' - // to be compatible with AWS behavior - assert.strictEqual(res.ContentEncoding, 'gzip,'); - assert.strictEqual(res.Expires.toGMTString(), - newExpires.toGMTString()); - done(); - }).catch(err => { - checkNoError(err); - done(err); - }); - }).catch(err => { - checkNoError(err); - done(err); - }); - }); + it( + 'should also replace additional headers if replace ' + + 'included as metadata directive header and new headers are ' + + 'specified', + done => { + s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + MetadataDirective: 'REPLACE', + CacheControl: newCacheControl, + ContentDisposition: newContentDisposition, + ContentEncoding: newContentEncoding, + Expires: newExpires, + }), + ) + .then(() => { + s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })) + .then(res => { + assert.strictEqual(res.CacheControl, newCacheControl); + assert.strictEqual(res.ContentDisposition, newContentDisposition); + // Should remove V4 streaming value 'aws-chunked' + // to be compatible with AWS behavior + assert.strictEqual(res.ContentEncoding, 'gzip,'); + assert.strictEqual(res.Expires.toGMTString(), newExpires.toGMTString()); + done(); + }) + .catch(err => { + checkNoError(err); + done(err); + }); + }) + .catch(err => { + checkNoError(err); + done(err); + }); + }, + ); - it('should copy an object and the metadata if copy ' + - 'included as metadata directive header (and ignore any new ' + - 'metadata sent with copy request)', done => { - s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - MetadataDirective: 'COPY', - Metadata: newMetadata, - })).then(() => { - s3.send(new GetObjectCommand({ Bucket: destBucketName, - Key: destObjName })).then(res => { - assert.deepStrictEqual(res.Metadata, originalMetadata); - done(); - }).catch(err => { + it( + 'should copy an object and the metadata if copy ' + + 'included as metadata directive header (and ignore any new ' + + 'metadata sent with copy request)', + done => { + s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + MetadataDirective: 'COPY', + Metadata: newMetadata, + }), + ) + .then(() => { + s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })) + .then(res => { + assert.deepStrictEqual(res.Metadata, originalMetadata); + done(); + }) + .catch(err => { + checkNoError(err); + done(err); + }); + }) + .catch(err => { checkNoError(err); done(err); }); - }).catch(err => { - checkNoError(err); - done(err); - }); - }); + }, + ); - it('should copy an object and its additional headers if copy ' + - 'included as metadata directive header (and ignore any new ' + - 'headers sent with copy request)', done => { - s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - MetadataDirective: 'COPY', - Metadata: newMetadata, - CacheControl: newCacheControl, - ContentDisposition: newContentDisposition, - ContentEncoding: newContentEncoding, - Expires: newExpires, - })).then(() => { - s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })).then(res => { - assert.strictEqual(res.CacheControl, - originalCacheControl); - assert.strictEqual(res.ContentDisposition, - originalContentDisposition); - assert.strictEqual(res.ContentEncoding, - 'base64,'); - assert.strictEqual(res.Expires.toGMTString(), - originalExpires.toGMTString()); - done(); - }); - }); - }); + it( + 'should copy an object and its additional headers if copy ' + + 'included as metadata directive header (and ignore any new ' + + 'headers sent with copy request)', + done => { + s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + MetadataDirective: 'COPY', + Metadata: newMetadata, + CacheControl: newCacheControl, + ContentDisposition: newContentDisposition, + ContentEncoding: newContentEncoding, + Expires: newExpires, + }), + ).then(() => { + s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })).then(res => { + assert.strictEqual(res.CacheControl, originalCacheControl); + assert.strictEqual(res.ContentDisposition, originalContentDisposition); + assert.strictEqual(res.ContentEncoding, 'base64,'); + assert.strictEqual(res.Expires.toGMTString(), originalExpires.toGMTString()); + done(); + }); + }); + }, + ); it('should copy a 0 byte object to different destination', done => { const emptyFileETag = '"d41d8cd98f00b204e9800998ecf8427e"'; - s3.send(new PutObjectCommand({ Bucket: sourceBucketName, Key: sourceObjName, - Body: '', Metadata: originalMetadata })).then(() => { - s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - })).then(res => { - assert.strictEqual(res.CopyObjectResult.ETag, emptyFileETag); - s3.send(new GetObjectCommand({ Bucket: destBucketName, - Key: destObjName })).then(res => { - assert.deepStrictEqual(res.Metadata, - originalMetadata); - assert.strictEqual(res.ETag, emptyFileETag); - done(); + s3.send( + new PutObjectCommand({ + Bucket: sourceBucketName, + Key: sourceObjName, + Body: '', + Metadata: originalMetadata, + }), + ) + .then(() => { + s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + }), + ) + .then(res => { + assert.strictEqual(res.CopyObjectResult.ETag, emptyFileETag); + s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })).then(res => { + assert.deepStrictEqual(res.Metadata, originalMetadata); + assert.strictEqual(res.ETag, emptyFileETag); + done(); + }); + }) + .catch(err => { + checkNoError(err); + done(err); }); - }).catch(err => { - checkNoError(err); - done(err); - }); - }).catch(err => { - checkNoError(err); - done(err); - }); + }) + .catch(err => { + checkNoError(err); + done(err); + }); }); // TODO: remove (or update to use different location constraint) in CLDSRV-639 @@ -576,403 +760,554 @@ describe('Object Copy', () => { it('should copy a 0 byte object to same destination', done => { const emptyFileETag = '"d41d8cd98f00b204e9800998ecf8427e"'; s3.send(new PutObjectCommand({ Bucket: sourceBucketName, Key: sourceObjName, Body: '' })).then(() => { - s3.send(new CopyObjectCommand({ Bucket: sourceBucketName, Key: sourceObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - StorageClass: 'REDUCED_REDUNDANCY', - })).then(res => { - assert.strictEqual(res.CopyObjectResult.ETag, emptyFileETag); - s3.send(new GetObjectCommand({ Bucket: sourceBucketName, - Key: sourceObjName })).then(res => { - assert.deepStrictEqual(res.Metadata, - {}); - assert.deepStrictEqual(res.StorageClass, - 'REDUCED_REDUNDANCY'); - assert.strictEqual(res.ETag, emptyFileETag); - done(); - }).catch(err => { + s3.send( + new CopyObjectCommand({ + Bucket: sourceBucketName, + Key: sourceObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + StorageClass: 'REDUCED_REDUNDANCY', + }), + ) + .then(res => { + assert.strictEqual(res.CopyObjectResult.ETag, emptyFileETag); + s3.send(new GetObjectCommand({ Bucket: sourceBucketName, Key: sourceObjName })) + .then(res => { + assert.deepStrictEqual(res.Metadata, {}); + assert.deepStrictEqual(res.StorageClass, 'REDUCED_REDUNDANCY'); + assert.strictEqual(res.ETag, emptyFileETag); + done(); + }) + .catch(err => { + checkNoError(err); + done(err); + }); + }) + .catch(err => { checkNoError(err); done(err); }); - }).catch(err => { - checkNoError(err); - done(err); - }); }); }); - it('should copy an object to a different destination and change ' + - 'the storage class if storage class header provided', done => { - s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - StorageClass: 'REDUCED_REDUNDANCY', - })).then(() => { - s3.send(new GetObjectCommand({ Bucket: destBucketName, - Key: destObjName })).then(res => { - assert.strictEqual(res.StorageClass, - 'REDUCED_REDUNDANCY'); - done(); - }).catch(err => { + it( + 'should copy an object to a different destination and change ' + + 'the storage class if storage class header provided', + done => { + s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + StorageClass: 'REDUCED_REDUNDANCY', + }), + ) + .then(() => { + s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })) + .then(res => { + assert.strictEqual(res.StorageClass, 'REDUCED_REDUNDANCY'); + done(); + }) + .catch(err => { + checkNoError(err); + done(err); + }); + }) + .catch(err => { + checkNoError(err); + done(err); + }); + }, + ); + + it( + 'should copy an object to the same destination and change the ' + + 'storage class if the storage class header provided', + done => { + s3.send( + new CopyObjectCommand({ + Bucket: sourceBucketName, + Key: sourceObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + StorageClass: 'REDUCED_REDUNDANCY', + }), + ) + .then(() => { + s3.send(new GetObjectCommand({ Bucket: sourceBucketName, Key: sourceObjName })) + .then(res => { + assert.strictEqual(res.StorageClass, 'REDUCED_REDUNDANCY'); + done(); + }) + .catch(err => { + checkNoError(err); + done(err); + }); + }) + .catch(err => { + checkNoError(err); + done(err); + }); + }, + ); + } + + it( + 'should copy an object to a new bucket and overwrite an already ' + + 'existing object in the destination bucket', + done => { + s3.send( + new PutObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + Body: 'overwrite me', + Metadata: originalMetadata, + }), + ) + .then(() => { + s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + MetadataDirective: 'REPLACE', + Metadata: newMetadata, + }), + ) + .then(res => { + assert.strictEqual(res.CopyObjectResult.ETag, etag); + s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })) + .then(async res => { + assert.deepStrictEqual(res.Metadata, newMetadata); + assert.strictEqual(res.ETag, etag); + const bodyString = await res.Body.transformToString(); + assert.strictEqual(bodyString, content); + done(); + }) + .catch(err => { + checkNoError(err); + done(err); + }); + }) + .catch(err => { + checkNoError(err); + done(err); + }); + }) + .catch(err => { checkNoError(err); done(err); }); - }).catch(err => { - checkNoError(err); - done(err); - }); - }); + }, + ); - it('should copy an object to the same destination and change the ' + - 'storage class if the storage class header provided', done => { - s3.send(new CopyObjectCommand({ Bucket: sourceBucketName, Key: sourceObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - StorageClass: 'REDUCED_REDUNDANCY', - })).then(() => { - s3.send(new GetObjectCommand({ Bucket: sourceBucketName, - Key: sourceObjName })).then(res => { - assert.strictEqual(res.StorageClass, - 'REDUCED_REDUNDANCY'); - done(); - }).catch(err => { + // skipping test as object level encryption is not implemented yet + it.skip( + 'should copy an object and change the server side encryption' + + 'option if server side encryption header provided', + done => { + s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + ServerSideEncryption: 'AES256', + }), + ) + .then(() => { + s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })) + .then(res => { + assert.strictEqual(res.ServerSideEncryption, 'AES256'); + done(); + }) + .catch(err => { + checkNoError(err); + done(err); + }); + }) + .catch(err => { checkNoError(err); done(err); }); - }).catch(err => { - checkNoError(err); - done(err); - }); - }); - } + }, + ); - it('should copy an object to a new bucket and overwrite an already ' + - 'existing object in the destination bucket', done => { - s3.send(new PutObjectCommand({ Bucket: destBucketName, Key: destObjName, - Body: 'overwrite me', Metadata: originalMetadata })).then(() => { - s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, + it( + 'should return Not Implemented error for obj. encryption using ' + 'customer-provided encryption keys', + done => { + const params = { + Bucket: destBucketName, + Key: 'key', CopySource: `${sourceBucketName}/${sourceObjName}`, - MetadataDirective: 'REPLACE', - Metadata: newMetadata, - })).then(res => { - assert.strictEqual(res.CopyObjectResult.ETag, etag); - s3.send(new GetObjectCommand({ Bucket: destBucketName, - Key: destObjName })).then(async res => { - assert.deepStrictEqual(res.Metadata, - newMetadata); - assert.strictEqual(res.ETag, etag); - const bodyString = await res.Body.transformToString(); - assert.strictEqual(bodyString, content); - done(); - }).catch(err => { - checkNoError(err); - done(err); - }); - }).catch(err => { - checkNoError(err); - done(err); - }); - }).catch(err => { - checkNoError(err); - done(err); - } - ); - }); - - // skipping test as object level encryption is not implemented yet - it.skip('should copy an object and change the server side encryption' + - 'option if server side encryption header provided', done => { - s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - ServerSideEncryption: 'AES256', - })).then(() => { - s3.send(new GetObjectCommand({ Bucket: destBucketName, - Key: destObjName })).then(res => { - assert.strictEqual(res.ServerSideEncryption, - 'AES256'); - done(); - }).catch(err => { - checkNoError(err); - done(err); - }); - }).catch(err => { - checkNoError(err); - done(err); - }); - }); - - it('should return Not Implemented error for obj. encryption using ' + - 'customer-provided encryption keys', done => { - const params = { Bucket: destBucketName, Key: 'key', - CopySource: `${sourceBucketName}/${sourceObjName}`, - SSECustomerAlgorithm: 'AES256' }; - s3.send(new CopyObjectCommand(params)).then(() => { - throw Error('Expected NotImplemented error'); - }).catch(err => { - assert.strictEqual(err.name, 'NotImplemented'); - done(); - }); - }); - - it('should copy an object and set the acl on the new object', done => { - s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - ACL: 'authenticated-read', - })).then(() => { - s3.send(new GetObjectAclCommand({ Bucket: destBucketName, - Key: destObjName })).then(res => { - // With authenticated-read ACL, there are two - // grants: - // (1) FULL_CONTROL to the object owner - // (2) READ to the authenticated-read - assert.strictEqual(res.Grants.length, 2); - assert.strictEqual(res.Grants[0].Permission, - 'FULL_CONTROL'); - assert.strictEqual(res.Grants[1].Permission, - 'READ'); - assert.strictEqual(res.Grants[1].Grantee.URI, - 'http://acs.amazonaws.com/groups/' + - 'global/AuthenticatedUsers'); + SSECustomerAlgorithm: 'AES256', + }; + s3.send(new CopyObjectCommand(params)) + .then(() => { + throw Error('Expected NotImplemented error'); + }) + .catch(err => { + assert.strictEqual(err.name, 'NotImplemented'); done(); - }).catch(err => { - checkNoError(err); - done(err); }); - }).catch(err => { - checkNoError(err); - done(err); - }); - }); + }, + ); - it('should copy an object and default the acl on the new object ' + - 'to private even if the copied object had a ' + - 'different acl', done => { - s3.send(new PutObjectAclCommand({ Bucket: sourceBucketName, Key: sourceObjName, - ACL: 'authenticated-read' })).then(() => { - s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, + it('should copy an object and set the acl on the new object', done => { + s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, CopySource: `${sourceBucketName}/${sourceObjName}`, - })).then(() => { - s3.send(new GetObjectAclCommand({ Bucket: destBucketName, - Key: destObjName })).then(res => { - // With private ACL, there is only one grant - // of FULL_CONTROL to the object owner - assert.strictEqual(res.Grants.length, 1); - assert.strictEqual(res.Grants[0].Permission, - 'FULL_CONTROL'); + ACL: 'authenticated-read', + }), + ) + .then(() => { + s3.send(new GetObjectAclCommand({ Bucket: destBucketName, Key: destObjName })) + .then(res => { + // With authenticated-read ACL, there are two + // grants: + // (1) FULL_CONTROL to the object owner + // (2) READ to the authenticated-read + assert.strictEqual(res.Grants.length, 2); + assert.strictEqual(res.Grants[0].Permission, 'FULL_CONTROL'); + assert.strictEqual(res.Grants[1].Permission, 'READ'); + assert.strictEqual( + res.Grants[1].Grantee.URI, + 'http://acs.amazonaws.com/groups/' + 'global/AuthenticatedUsers', + ); done(); - }).catch(err => { + }) + .catch(err => { checkNoError(err); done(err); }); - }).catch(err => { + }) + .catch(err => { checkNoError(err); done(err); }); - }).catch(err => { - checkNoError(err); - done(err); - }); }); - it('should return an error if attempt to copy with same source as' + - 'destination and do not change any metadata', done => { - s3.send(new CopyObjectCommand({ Bucket: sourceBucketName, Key: sourceObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - })).then(() => { - done(); - }).catch(err => { - checkError(err, 'InvalidRequest', 400); - done(); - }); - }); + it( + 'should copy an object and default the acl on the new object ' + + 'to private even if the copied object had a ' + + 'different acl', + done => { + s3.send( + new PutObjectAclCommand({ + Bucket: sourceBucketName, + Key: sourceObjName, + ACL: 'authenticated-read', + }), + ) + .then(() => { + s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + }), + ) + .then(() => { + s3.send(new GetObjectAclCommand({ Bucket: destBucketName, Key: destObjName })) + .then(res => { + // With private ACL, there is only one grant + // of FULL_CONTROL to the object owner + assert.strictEqual(res.Grants.length, 1); + assert.strictEqual(res.Grants[0].Permission, 'FULL_CONTROL'); + done(); + }) + .catch(err => { + checkNoError(err); + done(err); + }); + }) + .catch(err => { + checkNoError(err); + done(err); + }); + }) + .catch(err => { + checkNoError(err); + done(err); + }); + }, + ); - it('should return an error if attempt to copy from nonexistent bucket', + it( + 'should return an error if attempt to copy with same source as' + + 'destination and do not change any metadata', done => { - s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, + s3.send( + new CopyObjectCommand({ + Bucket: sourceBucketName, + Key: sourceObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + }), + ) + .then(() => { + done(); + }) + .catch(err => { + checkError(err, 'InvalidRequest', 400); + done(); + }); + }, + ); + + it('should return an error if attempt to copy from nonexistent bucket', done => { + s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, CopySource: `nobucket453234/${sourceObjName}`, - })).then(() => { + }), + ) + .then(() => { done(); - }).catch(err => { + }) + .catch(err => { checkError(err, 'NoSuchBucket', 404); done(); }); - }); + }); - it('should return an error if use invalid redirect location', - done => { - s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, + it('should return an error if use invalid redirect location', done => { + s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, CopySource: `${sourceBucketName}/${sourceObjName}`, WebsiteRedirectLocation: 'google.com', - })).then(() => { + }), + ) + .then(() => { done(); - }).catch(err => { + }) + .catch(err => { checkError(err, 'InvalidRedirectLocation', 400); done(); }); - }); + }); - it('should return an error if copy request has object lock legal ' + - 'hold header but object lock is not enabled on destination bucket', + it( + 'should return an error if copy request has object lock legal ' + + 'hold header but object lock is not enabled on destination bucket', done => { - s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - ObjectLockLegalHoldStatus: 'ON', - })).then(() => { - done(); - }).catch(err => { - checkError(err, 'InvalidRequest', 400); - done(); - }); - }); + s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + ObjectLockLegalHoldStatus: 'ON', + }), + ) + .then(() => { + done(); + }) + .catch(err => { + checkError(err, 'InvalidRequest', 400); + done(); + }); + }, + ); - it('should return an error if copy request has retention headers ' + - 'but object lock is not enabled on destination bucket', + it( + 'should return an error if copy request has retention headers ' + + 'but object lock is not enabled on destination bucket', done => { const mockDate = new Date(2050, 10, 12); - s3.send(new CopyObjectCommand({ - Bucket: destBucketName, - Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - ObjectLockMode: 'GOVERNANCE', - ObjectLockRetainUntilDate: mockDate, - })).then(() => { - done(); - }).catch(err => { - checkError(err, 'InvalidRequest', 400); - done(); - }); - }); + s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + ObjectLockMode: 'GOVERNANCE', + ObjectLockRetainUntilDate: mockDate, + }), + ) + .then(() => { + done(); + }) + .catch(err => { + checkError(err, 'InvalidRequest', 400); + done(); + }); + }, + ); - it('should return an error if attempt to copy to nonexistent bucket', - done => { - s3.send(new CopyObjectCommand({ Bucket: 'nobucket453234', Key: destObjName, + it('should return an error if attempt to copy to nonexistent bucket', done => { + s3.send( + new CopyObjectCommand({ + Bucket: 'nobucket453234', + Key: destObjName, CopySource: `${sourceBucketName}/${sourceObjName}`, - })).then(() => { + }), + ) + .then(() => { done(); - }).catch(err => { + }) + .catch(err => { checkError(err, 'NoSuchBucket', 404); done(); }); - }); + }); - it('should return an error if attempt to copy nonexistent object', - done => { - s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, + it('should return an error if attempt to copy nonexistent object', done => { + s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, CopySource: `${sourceBucketName}/nokey`, - })).then(() => { + }), + ) + .then(() => { done(); - }).catch(err => { + }) + .catch(err => { checkError(err, 'NoSuchKey', 404); done(); }); - }); + }); - it('should return an error if attempt to copy nonexistent object', - done => { - s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, + it('should return an error if attempt to copy nonexistent object', done => { + s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, CopySource: `${sourceBucketName}/nokey`, - })).then(() => { + }), + ) + .then(() => { done(); - }).catch(err => { + }) + .catch(err => { checkError(err, 'NoSuchKey', 404); done(); }); - }); + }); - it('should return an error if send invalid metadata directive header', - done => { - s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, + it('should return an error if send invalid metadata directive header', done => { + s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, CopySource: `${sourceBucketName}/${sourceObjName}`, MetadataDirective: 'copyHalf', - })).then(() => { + }), + ) + .then(() => { done(); - }).catch(err => { + }) + .catch(err => { checkError(err, 'InvalidArgument', 400); done(); }); - }); + }); describe('copying by another account', () => { const otherAccountBucket = 'otheraccountbucket42342342342'; const otherAccountKey = 'key'; - beforeEach(() => otherAccountBucketUtility - .createOne(otherAccountBucket) + beforeEach(() => otherAccountBucketUtility.createOne(otherAccountBucket)); + + afterEach(() => + otherAccountBucketUtility + .empty(otherAccountBucket) + .then(() => otherAccountBucketUtility.deleteOne(otherAccountBucket)), ); - afterEach(() => otherAccountBucketUtility.empty(otherAccountBucket) - .then(() => otherAccountBucketUtility - .deleteOne(otherAccountBucket)) + it( + 'should not allow an account without read persmission on the ' + 'source object to copy the object', + done => { + otherAccountS3 + .send( + new CopyObjectCommand({ + Bucket: otherAccountBucket, + Key: otherAccountKey, + CopySource: `${sourceBucketName}/${sourceObjName}`, + }), + ) + .then(() => { + done(); + }) + .catch(err => { + checkError(err, 'AccessDenied', 403); + done(); + }); + }, ); - it('should not allow an account without read persmission on the ' + - 'source object to copy the object', done => { - otherAccountS3.send(new CopyObjectCommand({ Bucket: otherAccountBucket, - Key: otherAccountKey, - CopySource: `${sourceBucketName}/${sourceObjName}`, - })).then(() => { - done(); - }).catch(err => { - checkError(err, 'AccessDenied', 403); - done(); - }); - }); + it( + 'should not allow an account without write persmission on the ' + + 'destination bucket to copy the object', + () => + otherAccountS3 + .send(new PutObjectCommand({ Bucket: otherAccountBucket, Key: otherAccountKey, Body: '' })) + .then(() => + otherAccountS3 + .send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${otherAccountBucket}/${otherAccountKey}`, + }), + ) + .catch(err => { + checkError(err, 'AccessDenied', 403); + }), + ), + ); - it('should not allow an account without write persmission on the ' + - 'destination bucket to copy the object', () => otherAccountS3.send(new PutObjectCommand( - { Bucket: otherAccountBucket, - Key: otherAccountKey, Body: '' })).then(() => otherAccountS3.send(new CopyObjectCommand( - { Bucket: destBucketName, - Key: destObjName, - CopySource: `${otherAccountBucket}/${otherAccountKey}`, - })).catch(err => { - checkError(err, 'AccessDenied', 403); - }))); - - - it('should allow an account with read permission on the ' + - 'source object and write permission on the destination ' + - 'bucket to copy the object', () => s3.send(new PutObjectAclCommand({ Bucket: sourceBucketName, - Key: sourceObjName, ACL: 'public-read' })).then(() => otherAccountS3.send(new CopyObjectCommand( - { Bucket: otherAccountBucket, - Key: otherAccountKey, - CopySource: `${sourceBucketName}/${sourceObjName}`, - })))); + it( + 'should allow an account with read permission on the ' + + 'source object and write permission on the destination ' + + 'bucket to copy the object', + () => + s3 + .send( + new PutObjectAclCommand({ + Bucket: sourceBucketName, + Key: sourceObjName, + ACL: 'public-read', + }), + ) + .then(() => + otherAccountS3.send( + new CopyObjectCommand({ + Bucket: otherAccountBucket, + Key: otherAccountKey, + CopySource: `${sourceBucketName}/${sourceObjName}`, + }), + ), + ), + ); }); - it('If-Match: returns no error when ETag match, with double quotes ' + - 'around ETag', - done => { - requestCopy({ CopySourceIfMatch: etag }, err => { - checkNoError(err); - done(); - }); + it('If-Match: returns no error when ETag match, with double quotes ' + 'around ETag', done => { + requestCopy({ CopySourceIfMatch: etag }, err => { + checkNoError(err); + done(); }); + }); - it('If-Match: returns no error when one of ETags match, with double ' + - 'quotes around ETag', - done => { - requestCopy({ CopySourceIfMatch: - `non-matching,${etag}` }, err => { - checkNoError(err); - done(); - }); + it('If-Match: returns no error when one of ETags match, with double ' + 'quotes around ETag', done => { + requestCopy({ CopySourceIfMatch: `non-matching,${etag}` }, err => { + checkNoError(err); + done(); }); + }); - it('If-Match: returns no error when ETag match, without double ' + - 'quotes around ETag', - done => { - requestCopy({ CopySourceIfMatch: etagTrim }, err => { - checkNoError(err); - done(); - }); + it('If-Match: returns no error when ETag match, without double ' + 'quotes around ETag', done => { + requestCopy({ CopySourceIfMatch: etagTrim }, err => { + checkNoError(err); + done(); }); + }); - it('If-Match: returns no error when one of ETags match, without ' + - 'double quotes around ETag', - done => { - requestCopy({ CopySourceIfMatch: - `non-matching,${etagTrim}` }, err => { - checkNoError(err); - done(); - }); + it('If-Match: returns no error when one of ETags match, without ' + 'double quotes around ETag', done => { + requestCopy({ CopySourceIfMatch: `non-matching,${etagTrim}` }, err => { + checkNoError(err); + done(); }); + }); it('If-Match: returns no error when ETag match with *', done => { requestCopy({ CopySourceIfMatch: '*' }, err => { @@ -981,13 +1316,12 @@ describe('Object Copy', () => { }); }); - it('If-Match: returns PreconditionFailed when ETag does not match', - done => { - requestCopy({ CopySourceIfMatch: 'non-matching ETag' }, err => { - checkError(err, 'PreconditionFailed', 412); - done(); - }); + it('If-Match: returns PreconditionFailed when ETag does not match', done => { + requestCopy({ CopySourceIfMatch: 'non-matching ETag' }, err => { + checkError(err, 'PreconditionFailed', 412); + done(); }); + }); it('If-None-Match: returns no error when ETag does not match', done => { requestCopy({ CopySourceIfNoneMatch: 'non-matching' }, err => { @@ -996,342 +1330,402 @@ describe('Object Copy', () => { }); }); - it('If-None-Match: returns no error when all ETags do not match', - done => { - requestCopy({ + it('If-None-Match: returns no error when all ETags do not match', done => { + requestCopy( + { CopySourceIfNoneMatch: 'non-matching,non-matching-either', - }, err => { + }, + err => { checkNoError(err); done(); - }); - }); + }, + ); + }); - it('If-None-Match: returns PreconditionFailed when ETag match, with' + - 'double quotes around ETag', - done => { - requestCopy({ CopySourceIfNoneMatch: etag }, err => { - checkError(err, 'PreconditionFailed', 412); - done(); - }); + it('If-None-Match: returns PreconditionFailed when ETag match, with' + 'double quotes around ETag', done => { + requestCopy({ CopySourceIfNoneMatch: etag }, err => { + checkError(err, 'PreconditionFailed', 412); + done(); }); + }); - it('If-None-Match: returns PreconditionFailed when one of ETags ' + - 'match, with double quotes around ETag', + it( + 'If-None-Match: returns PreconditionFailed when one of ETags ' + 'match, with double quotes around ETag', done => { - requestCopy({ - CopySourceIfNoneMatch: `non-matching,${etag}`, - }, err => { - checkError(err, 'PreconditionFailed', 412); - done(); - }); - }); + requestCopy( + { + CopySourceIfNoneMatch: `non-matching,${etag}`, + }, + err => { + checkError(err, 'PreconditionFailed', 412); + done(); + }, + ); + }, + ); - it('If-None-Match: returns PreconditionFailed when ETag match, ' + - 'without double quotes around ETag', + it( + 'If-None-Match: returns PreconditionFailed when ETag match, ' + 'without double quotes around ETag', done => { requestCopy({ CopySourceIfNoneMatch: etagTrim }, err => { checkError(err, 'PreconditionFailed', 412); done(); }); - }); - - it('If-None-Match: returns PreconditionFailed when one of ETags ' + - 'match, without double quotes around ETag', - done => { - requestCopy({ - CopySourceIfNoneMatch: `non-matching,${etagTrim}`, - }, err => { - checkError(err, 'PreconditionFailed', 412); - done(); - }); - }); + }, + ); - it('If-Modified-Since: returns no error if Last modified date is ' + - 'greater', + it( + 'If-None-Match: returns PreconditionFailed when one of ETags ' + 'match, without double quotes around ETag', done => { - requestCopy({ CopySourceIfModifiedSince: dateFromNow(-1) }, + requestCopy( + { + CopySourceIfNoneMatch: `non-matching,${etagTrim}`, + }, err => { - checkNoError(err); + checkError(err, 'PreconditionFailed', 412); done(); - }); + }, + ); + }, + ); + + it('If-Modified-Since: returns no error if Last modified date is ' + 'greater', done => { + requestCopy({ CopySourceIfModifiedSince: dateFromNow(-1) }, err => { + checkNoError(err); + done(); }); + }); // Skipping this test, because real AWS does not provide error as // expected - it.skip('If-Modified-Since: returns PreconditionFailed if Last ' + - 'modified date is lesser', - done => { - requestCopy({ CopySourceIfModifiedSince: dateFromNow(1) }, - err => { - checkError(err, 'PreconditionFailed', 412); - done(); - }); + it.skip('If-Modified-Since: returns PreconditionFailed if Last ' + 'modified date is lesser', done => { + requestCopy({ CopySourceIfModifiedSince: dateFromNow(1) }, err => { + checkError(err, 'PreconditionFailed', 412); + done(); }); + }); - it('If-Modified-Since: returns PreconditionFailed if Last modified ' + - 'date is equal', - done => { - requestCopy({ CopySourceIfModifiedSince: - dateConvert(lastModified) }, - err => { - checkError(err, 'PreconditionFailed', 412); - done(); - }); + it('If-Modified-Since: returns PreconditionFailed if Last modified ' + 'date is equal', done => { + requestCopy({ CopySourceIfModifiedSince: dateConvert(lastModified) }, err => { + checkError(err, 'PreconditionFailed', 412); + done(); }); + }); - it('If-Unmodified-Since: returns no error when lastModified date is ' + - 'greater', - done => { - requestCopy({ CopySourceIfUnmodifiedSince: dateFromNow(1) }, - err => { - checkNoError(err); - done(); - }); + it('If-Unmodified-Since: returns no error when lastModified date is ' + 'greater', done => { + requestCopy({ CopySourceIfUnmodifiedSince: dateFromNow(1) }, err => { + checkNoError(err); + done(); }); + }); - it('If-Unmodified-Since: returns no error when lastModified ' + - 'date is equal', - done => { - requestCopy({ CopySourceIfUnmodifiedSince: - dateConvert(lastModified) }, - err => { - checkNoError(err); - done(); - }); + it('If-Unmodified-Since: returns no error when lastModified ' + 'date is equal', done => { + requestCopy({ CopySourceIfUnmodifiedSince: dateConvert(lastModified) }, err => { + checkNoError(err); + done(); }); + }); - it('If-Unmodified-Since: returns PreconditionFailed when ' + - 'lastModified date is lesser', - done => { - requestCopy({ CopySourceIfUnmodifiedSince: dateFromNow(-1) }, - err => { - checkError(err, 'PreconditionFailed', 412); - done(); - }); + it('If-Unmodified-Since: returns PreconditionFailed when ' + 'lastModified date is lesser', done => { + requestCopy({ CopySourceIfUnmodifiedSince: dateFromNow(-1) }, err => { + checkError(err, 'PreconditionFailed', 412); + done(); }); + }); - it('If-Match & If-Unmodified-Since: returns no error when match Etag ' + - 'and lastModified is greater', + it( + 'If-Match & If-Unmodified-Since: returns no error when match Etag ' + 'and lastModified is greater', done => { - requestCopy({ + requestCopy( + { + CopySourceIfMatch: etagTrim, + CopySourceIfUnmodifiedSince: dateFromNow(-1), + }, + err => { + checkNoError(err); + done(); + }, + ); + }, + ); + + it('If-Match match & If-Unmodified-Since match', done => { + requestCopy( + { CopySourceIfMatch: etagTrim, - CopySourceIfUnmodifiedSince: dateFromNow(-1), - }, err => { + CopySourceIfUnmodifiedSince: dateFromNow(1), + }, + err => { checkNoError(err); done(); - }); - }); - - it('If-Match match & If-Unmodified-Since match', done => { - requestCopy({ - CopySourceIfMatch: etagTrim, - CopySourceIfUnmodifiedSince: dateFromNow(1), - }, err => { - checkNoError(err); - done(); - }); + }, + ); }); it('If-Match not match & If-Unmodified-Since not match', done => { - requestCopy({ - CopySourceIfMatch: 'non-matching', - CopySourceIfUnmodifiedSince: dateFromNow(-1), - }, err => { - checkError(err, 'PreconditionFailed', 412); - done(); - }); + requestCopy( + { + CopySourceIfMatch: 'non-matching', + CopySourceIfUnmodifiedSince: dateFromNow(-1), + }, + err => { + checkError(err, 'PreconditionFailed', 412); + done(); + }, + ); }); it('If-Match not match & If-Unmodified-Since match', done => { - requestCopy({ - CopySourceIfMatch: 'non-matching', - CopySourceIfUnmodifiedSince: dateFromNow(1), - }, err => { - checkError(err, 'PreconditionFailed'); - done(); - }); + requestCopy( + { + CopySourceIfMatch: 'non-matching', + CopySourceIfUnmodifiedSince: dateFromNow(1), + }, + err => { + checkError(err, 'PreconditionFailed'); + done(); + }, + ); }); // Skipping this test, because real AWS does not provide error as // expected it.skip('If-Match match & If-Modified-Since not match', done => { - requestCopy({ - CopySourceIfMatch: etagTrim, - CopySourceIfModifiedSince: dateFromNow(1), - }, err => { - checkNoError(err); - done(); - }); + requestCopy( + { + CopySourceIfMatch: etagTrim, + CopySourceIfModifiedSince: dateFromNow(1), + }, + err => { + checkNoError(err); + done(); + }, + ); }); it('If-Match match & If-Modified-Since match', done => { - requestCopy({ - CopySourceIfMatch: etagTrim, - CopySourceIfModifiedSince: dateFromNow(-1), - }, err => { - checkNoError(err); - done(); - }); + requestCopy( + { + CopySourceIfMatch: etagTrim, + CopySourceIfModifiedSince: dateFromNow(-1), + }, + err => { + checkNoError(err); + done(); + }, + ); }); it('If-Match not match & If-Modified-Since not match', done => { - requestCopy({ - CopySourceIfMatch: 'non-matching', - CopySourceIfModifiedSince: dateFromNow(1), - }, err => { - checkError(err, 'PreconditionFailed', 412); - done(); - }); + requestCopy( + { + CopySourceIfMatch: 'non-matching', + CopySourceIfModifiedSince: dateFromNow(1), + }, + err => { + checkError(err, 'PreconditionFailed', 412); + done(); + }, + ); }); it('If-Match not match & If-Modified-Since match', done => { - requestCopy({ - CopySourceIfMatch: 'non-matching', - CopySourceIfModifiedSince: dateFromNow(-1), - }, err => { - checkError(err, 'PreconditionFailed', 412); - done(); - }); + requestCopy( + { + CopySourceIfMatch: 'non-matching', + CopySourceIfModifiedSince: dateFromNow(-1), + }, + err => { + checkError(err, 'PreconditionFailed', 412); + done(); + }, + ); }); - it('If-None-Match & If-Modified-Since: returns PreconditionFailed ' + - 'when Etag does not match and lastModified is greater', + it( + 'If-None-Match & If-Modified-Since: returns PreconditionFailed ' + + 'when Etag does not match and lastModified is greater', done => { - requestCopy({ + requestCopy( + { + CopySourceIfNoneMatch: etagTrim, + CopySourceIfModifiedSince: dateFromNow(-1), + }, + err => { + checkError(err, 'PreconditionFailed', 412); + done(); + }, + ); + }, + ); + + it('If-None-Match not match & If-Modified-Since not match', done => { + requestCopy( + { CopySourceIfNoneMatch: etagTrim, - CopySourceIfModifiedSince: dateFromNow(-1), - }, err => { + CopySourceIfModifiedSince: dateFromNow(1), + }, + err => { checkError(err, 'PreconditionFailed', 412); done(); - }); - }); - - it('If-None-Match not match & If-Modified-Since not match', done => { - requestCopy({ - CopySourceIfNoneMatch: etagTrim, - CopySourceIfModifiedSince: dateFromNow(1), - }, err => { - checkError(err, 'PreconditionFailed', 412); - done(); - }); + }, + ); }); it('If-None-Match match & If-Modified-Since match', done => { - requestCopy({ - CopySourceIfNoneMatch: 'non-matching', - CopySourceIfModifiedSince: dateFromNow(-1), - }, err => { - checkNoError(err); - done(); - }); + requestCopy( + { + CopySourceIfNoneMatch: 'non-matching', + CopySourceIfModifiedSince: dateFromNow(-1), + }, + err => { + checkNoError(err); + done(); + }, + ); }); // Skipping this test, because real AWS does not provide error as // expected it.skip('If-None-Match match & If-Modified-Since not match', done => { - requestCopy({ - CopySourceIfNoneMatch: 'non-matching', - CopySourceIfModifiedSince: dateFromNow(1), - }, err => { - checkError(err, 'PreconditionFailed', 412); - done(); - }); + requestCopy( + { + CopySourceIfNoneMatch: 'non-matching', + CopySourceIfModifiedSince: dateFromNow(1), + }, + err => { + checkError(err, 'PreconditionFailed', 412); + done(); + }, + ); }); it('If-None-Match match & If-Unmodified-Since match', done => { - requestCopy({ - CopySourceIfNoneMatch: 'non-matching', - CopySourceIfUnmodifiedSince: dateFromNow(1), - }, err => { - checkNoError(err); - done(); - }); + requestCopy( + { + CopySourceIfNoneMatch: 'non-matching', + CopySourceIfUnmodifiedSince: dateFromNow(1), + }, + err => { + checkNoError(err); + done(); + }, + ); }); it('If-None-Match match & If-Unmodified-Since not match', done => { - requestCopy({ - CopySourceIfNoneMatch: 'non-matching', - CopySourceIfUnmodifiedSince: dateFromNow(-1), - }, err => { - checkError(err, 'PreconditionFailed', 412); - done(); - }); + requestCopy( + { + CopySourceIfNoneMatch: 'non-matching', + CopySourceIfUnmodifiedSince: dateFromNow(-1), + }, + err => { + checkError(err, 'PreconditionFailed', 412); + done(); + }, + ); }); it('If-None-Match not match & If-Unmodified-Since match', done => { - requestCopy({ - CopySourceIfNoneMatch: etagTrim, - CopySourceIfUnmodifiedSince: dateFromNow(1), - }, err => { - checkError(err, 'PreconditionFailed', 412); - done(); - }); + requestCopy( + { + CopySourceIfNoneMatch: etagTrim, + CopySourceIfUnmodifiedSince: dateFromNow(1), + }, + err => { + checkError(err, 'PreconditionFailed', 412); + done(); + }, + ); }); it('If-None-Match not match & If-Unmodified-Since not match', done => { - requestCopy({ - CopySourceIfNoneMatch: etagTrim, - CopySourceIfUnmodifiedSince: dateFromNow(-1), - }, err => { - checkError(err, 'PreconditionFailed', 412); - done(); - }); + requestCopy( + { + CopySourceIfNoneMatch: etagTrim, + CopySourceIfUnmodifiedSince: dateFromNow(-1), + }, + err => { + checkError(err, 'PreconditionFailed', 412); + done(); + }, + ); }); - it('should return InvalidStorageClass error when x-amz-storage-class header is provided ' + - 'and not equal to STANDARD', done => { - s3.send(new CopyObjectCommand({ - Bucket: destBucketName, - Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - StorageClass: 'COLD', - })).then(() => { - throw new Error('Expected InvalidStorageClass error'); - }).catch(err => { - assert.strictEqual(err.name, 'InvalidStorageClass'); - assert.strictEqual(err.$metadata.httpStatusCode, 400); - done(); - }); - }); + it( + 'should return InvalidStorageClass error when x-amz-storage-class header is provided ' + + 'and not equal to STANDARD', + done => { + s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + StorageClass: 'COLD', + }), + ) + .then(() => { + throw new Error('Expected InvalidStorageClass error'); + }) + .catch(err => { + assert.strictEqual(err.name, 'InvalidStorageClass'); + assert.strictEqual(err.$metadata.httpStatusCode, 400); + done(); + }); + }, + ); it('should not copy a cold object', done => { const archive = { archiveInfo: { archiveId: '97a71dfe-49c1-4cca-840a-69199e0b0322', - archiveVersion: 5577006791947779 + archiveVersion: 5577006791947779, }, }; fakeMetadataArchive(sourceBucketName, sourceObjName, undefined, archive, err => { assert.ifError(err); - s3.send(new CopyObjectCommand({ - Bucket: destBucketName, - Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - })).then(() => { - throw new Error('Expected InvalidObjectState error'); - }).catch(err => { - assert.strictEqual(err.name, 'InvalidObjectState'); - assert.strictEqual(err.$metadata.httpStatusCode, 403); - done(); - }); + s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + }), + ) + .then(() => { + throw new Error('Expected InvalidObjectState error'); + }) + .catch(err => { + assert.strictEqual(err.name, 'InvalidObjectState'); + assert.strictEqual(err.$metadata.httpStatusCode, 403); + done(); + }); }); }); - it('should copy an object when it\'s transitioning to cold', done => { + it("should copy an object when it's transitioning to cold", done => { fakeMetadataTransition(sourceBucketName, sourceObjName, undefined, err => { assert.ifError(err); - s3.send(new CopyObjectCommand({ - Bucket: destBucketName, - Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - })).then(async res => { - await successCopyCheck(null, res.CopyObjectResult, originalMetadata, - destBucketName, destObjName); - done(); - }).catch(err => { - checkNoError(err); - done(); - }); + s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + }), + ) + .then(async res => { + await successCopyCheck( + null, + res.CopyObjectResult, + originalMetadata, + destBucketName, + destObjName, + ); + done(); + }) + .catch(err => { + checkNoError(err); + done(); + }); }); }); @@ -1341,30 +1735,37 @@ describe('Object Copy', () => { restoreRequestedAt: new Date(0), restoreRequestedDays: 5, restoreCompletedAt: new Date(10), - restoreWillExpireAt: new Date(10 + (5 * 24 * 60 * 60 * 1000)), + restoreWillExpireAt: new Date(10 + 5 * 24 * 60 * 60 * 1000), }; fakeMetadataArchive(sourceBucketName, sourceObjName, undefined, archiveCompleted, err => { assert.ifError(err); - s3.send(new CopyObjectCommand({ - Bucket: destBucketName, - Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - })).then(async res => { - await successCopyCheck(null, res.CopyObjectResult, originalMetadata, - destBucketName, destObjName); - done(); - }).catch(err => { - checkNoError(err); - done(); - }); + s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + }), + ) + .then(async res => { + await successCopyCheck( + null, + res.CopyObjectResult, + originalMetadata, + destBucketName, + destObjName, + ); + done(); + }) + .catch(err => { + checkNoError(err); + done(); + }); }); }); }); }); - -describe('Object Copy with object lock enabled on both destination ' + - 'bucket and source bucket', () => { +describe('Object Copy with object lock enabled on both destination ' + 'bucket and source bucket', () => { withV4(sigCfg => { let bucketUtil; let s3; @@ -1373,10 +1774,10 @@ describe('Object Copy with object lock enabled on both destination ' + before(() => { bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; - return bucketUtil.empty(sourceBucketName, true) + return bucketUtil + .empty(sourceBucketName, true) .then(() => bucketUtil.empty(destBucketName)) - .then(() => - bucketUtil.deleteMany([sourceBucketName, destBucketName])) + .then(() => bucketUtil.deleteMany([sourceBucketName, destBucketName])) .catch(err => { if (err.name !== 'NoSuchBucket') { process.stdout.write(`${err}\n`); @@ -1390,20 +1791,28 @@ describe('Object Copy with object lock enabled on both destination ' + }); }); - beforeEach(() => s3.send(new PutObjectCommand({ - Bucket: sourceBucketName, - Key: sourceObjName, - Body: content, - Metadata: originalMetadata, - ObjectLockMode: 'GOVERNANCE', - ObjectLockRetainUntilDate: new Date(2050, 1, 1), - })).then(res => { - versionId = res.VersionId; - s3.send(new HeadObjectCommand({ - Bucket: sourceBucketName, - Key: sourceObjName, - })); - })); + beforeEach(() => + s3 + .send( + new PutObjectCommand({ + Bucket: sourceBucketName, + Key: sourceObjName, + Body: content, + Metadata: originalMetadata, + ObjectLockMode: 'GOVERNANCE', + ObjectLockRetainUntilDate: new Date(2050, 1, 1), + }), + ) + .then(res => { + versionId = res.VersionId; + s3.send( + new HeadObjectCommand({ + Bucket: sourceBucketName, + Key: sourceObjName, + }), + ); + }), + ); afterEach(async () => { await bucketUtil.empty(sourceBucketName); @@ -1412,126 +1821,148 @@ describe('Object Copy with object lock enabled on both destination ' + after(async () => await bucketUtil.deleteMany([sourceBucketName, destBucketName])); - it('should not copy default retention info of the destination ' + - 'bucket if legal hold header is passed with copy object request', + it( + 'should not copy default retention info of the destination ' + + 'bucket if legal hold header is passed with copy object request', done => { - s3.send(new CopyObjectCommand({ - Bucket: destBucketName, - Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - ObjectLockLegalHoldStatus: 'ON', - })).then(() => { - s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })) - .then(res => { - assert.strictEqual(res.ObjectLockMode, undefined); - assert.strictEqual(res.ObjectLockRetainUntilDate, - undefined); - assert.strictEqual(res.ObjectLockLegalHoldStatus, - 'ON'); - const removeLockObjs = [ - { - bucket: sourceBucketName, - key: sourceObjName, - versionId, - }, { - bucket: destBucketName, - key: destObjName, - versionId: res.VersionId, - }, - ]; - new Promise((resolve, reject) => { - changeObjectLock(removeLockObjs, '', err => { - if (err) { - reject(err); - } else { - resolve(); - } - }); - }).then(done).catch(err => { + s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + ObjectLockLegalHoldStatus: 'ON', + }), + ) + .then(() => { + s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })) + .then(res => { + assert.strictEqual(res.ObjectLockMode, undefined); + assert.strictEqual(res.ObjectLockRetainUntilDate, undefined); + assert.strictEqual(res.ObjectLockLegalHoldStatus, 'ON'); + const removeLockObjs = [ + { + bucket: sourceBucketName, + key: sourceObjName, + versionId, + }, + { + bucket: destBucketName, + key: destObjName, + versionId: res.VersionId, + }, + ]; + new Promise((resolve, reject) => { + changeObjectLock(removeLockObjs, '', err => { + if (err) { + reject(err); + } else { + resolve(); + } + }); + }) + .then(done) + .catch(err => { + assert.ifError(err); + done(err); + }); + }) + .catch(err => { assert.ifError(err); done(err); }); - }).catch(err => { - assert.ifError(err); - done(err); - }); - }).catch(err => { - assert.ifError(err); - done(err); - }); - }); + }) + .catch(err => { + assert.ifError(err); + done(err); + }); + }, + ); - it('should not copy default retention info of the destination ' + - 'bucket if legal hold header is passed with copy object request', + it( + 'should not copy default retention info of the destination ' + + 'bucket if legal hold header is passed with copy object request', done => { - s3.send(new CopyObjectCommand({ - Bucket: destBucketName, - Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - ObjectLockLegalHoldStatus: 'on', - })).then(() => { - s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })) - .then(res => { - assert.strictEqual(res.ObjectLockMode, undefined); - assert.strictEqual(res.ObjectLockMode, undefined); - assert.strictEqual(res.ObjectLockRetainUntilDate, - undefined); - assert.strictEqual(res.ObjectLockLegalHoldStatus, - 'OFF'); - const removeLockObjs = [ - { - bucket: sourceBucketName, - key: sourceObjName, - versionId, - }, - ]; - changeObjectLock(removeLockObjs, '', done); - }).catch(err => { - assert.ifError(err); - done(err); - }); - }).catch(err => { - assert.ifError(err); - done(err); - }); - }); + s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + ObjectLockLegalHoldStatus: 'on', + }), + ) + .then(() => { + s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })) + .then(res => { + assert.strictEqual(res.ObjectLockMode, undefined); + assert.strictEqual(res.ObjectLockMode, undefined); + assert.strictEqual(res.ObjectLockRetainUntilDate, undefined); + assert.strictEqual(res.ObjectLockLegalHoldStatus, 'OFF'); + const removeLockObjs = [ + { + bucket: sourceBucketName, + key: sourceObjName, + versionId, + }, + ]; + changeObjectLock(removeLockObjs, '', done); + }) + .catch(err => { + assert.ifError(err); + done(err); + }); + }) + .catch(err => { + assert.ifError(err); + done(err); + }); + }, + ); - it('should overwrite default retention info of the destination ' + - 'bucket if retention headers passed with copy object request', + it( + 'should overwrite default retention info of the destination ' + + 'bucket if retention headers passed with copy object request', done => { - s3.send(new CopyObjectCommand({ - Bucket: destBucketName, - Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - ObjectLockMode: 'COMPLIANCE', - ObjectLockRetainUntilDate: new Date(2055, 2, 3), - })).then(() => { - s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })) - .then(res => { - assert.strictEqual(res.ObjectLockMode, 'COMPLIANCE'); - assert.strictEqual(res.ObjectLockRetainUntilDate.toGMTString(), - new Date(2055, 2, 3).toGMTString()); - const removeLockObjs = [ - { - bucket: sourceBucketName, - key: sourceObjName, - versionId, - }, { - bucket: destBucketName, - key: destObjName, - versionId: res.VersionId, - }, - ]; - changeObjectLock(removeLockObjs, '', done); - }).catch(err => { - assert.ifError(err); - done(err); - }); - }).catch(err => { - assert.ifError(err); - done(err); - }); - }); - }); - + s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + ObjectLockMode: 'COMPLIANCE', + ObjectLockRetainUntilDate: new Date(2055, 2, 3), + }), + ) + .then(() => { + s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })) + .then(res => { + assert.strictEqual(res.ObjectLockMode, 'COMPLIANCE'); + assert.strictEqual( + res.ObjectLockRetainUntilDate.toGMTString(), + new Date(2055, 2, 3).toGMTString(), + ); + const removeLockObjs = [ + { + bucket: sourceBucketName, + key: sourceObjName, + versionId, + }, + { + bucket: destBucketName, + key: destObjName, + versionId: res.VersionId, + }, + ]; + changeObjectLock(removeLockObjs, '', done); + }) + .catch(err => { + assert.ifError(err); + done(err); + }); + }) + .catch(err => { + assert.ifError(err); + done(err); + }); + }, + ); + }); }); diff --git a/tests/functional/aws-node-sdk/test/object/objectGetAttributes.js b/tests/functional/aws-node-sdk/test/object/objectGetAttributes.js index 544b21c637..0b0a49ba58 100644 --- a/tests/functional/aws-node-sdk/test/object/objectGetAttributes.js +++ b/tests/functional/aws-node-sdk/test/object/objectGetAttributes.js @@ -40,12 +40,14 @@ describe('objectGetAttributes', () => { it('should fail with a wrong bucket owner header', async () => { try { - await s3.send(new GetObjectAttributesCommand({ - Bucket: bucket, - Key: key, - ObjectAttributes: ['ETag'], - ExpectedBucketOwner: 'wrongAccountId', - })); + await s3.send( + new GetObjectAttributesCommand({ + Bucket: bucket, + Key: key, + ObjectAttributes: ['ETag'], + ExpectedBucketOwner: 'wrongAccountId', + }), + ); assert.fail('Expected AccessDenied error'); } catch (err) { assert.strictEqual(err.name, 'AccessDenied'); @@ -55,11 +57,13 @@ describe('objectGetAttributes', () => { it('should fail because attributes header is missing', async () => { try { - await s3.send(new GetObjectAttributesCommand({ - Bucket: bucket, - Key: key, - ObjectAttributes: [], - })); + await s3.send( + new GetObjectAttributesCommand({ + Bucket: bucket, + Key: key, + ObjectAttributes: [], + }), + ); assert.fail('Expected InvalidArgument error'); } catch (err) { assert.strictEqual(err.name, 'InvalidArgument'); @@ -69,11 +73,13 @@ describe('objectGetAttributes', () => { it('should fail because attribute name is invalid', async () => { try { - await s3.send(new GetObjectAttributesCommand({ - Bucket: bucket, - Key: key, - ObjectAttributes: ['InvalidAttribute'], - })); + await s3.send( + new GetObjectAttributesCommand({ + Bucket: bucket, + Key: key, + ObjectAttributes: ['InvalidAttribute'], + }), + ); assert.fail('Expected InvalidArgument error'); } catch (err) { assert.strictEqual(err.name, 'InvalidArgument'); @@ -83,11 +89,13 @@ describe('objectGetAttributes', () => { it('should return NoSuchKey for non-existent object', async () => { try { - await s3.send(new GetObjectAttributesCommand({ - Bucket: bucket, - Key: 'nonexistent', - ObjectAttributes: ['ETag'], - })); + await s3.send( + new GetObjectAttributesCommand({ + Bucket: bucket, + Key: 'nonexistent', + ObjectAttributes: ['ETag'], + }), + ); assert.fail('Expected NoSuchKey error'); } catch (err) { assert.strictEqual(err.name, 'NoSuchKey'); @@ -96,11 +104,13 @@ describe('objectGetAttributes', () => { }); it('should return all attributes', async () => { - const data = await s3.send(new GetObjectAttributesCommand({ - Bucket: bucket, - Key: key, - ObjectAttributes: ['ETag', 'ObjectParts', 'StorageClass', 'ObjectSize'], - })); + const data = await s3.send( + new GetObjectAttributesCommand({ + Bucket: bucket, + Key: key, + ObjectAttributes: ['ETag', 'ObjectParts', 'StorageClass', 'ObjectSize'], + }), + ); assert.strictEqual(data.ETag, expectedMD5); assert.strictEqual(data.StorageClass, 'STANDARD'); @@ -110,22 +120,26 @@ describe('objectGetAttributes', () => { }); it('should return ETag', async () => { - const data = await s3.send(new GetObjectAttributesCommand({ - Bucket: bucket, - Key: key, - ObjectAttributes: ['ETag'], - })); + const data = await s3.send( + new GetObjectAttributesCommand({ + Bucket: bucket, + Key: key, + ObjectAttributes: ['ETag'], + }), + ); assert.strictEqual(data.ETag, expectedMD5); }); it('should fail with NotImplemented when Checksum is requested', async () => { try { - await s3.send(new GetObjectAttributesCommand({ - Bucket: bucket, - Key: key, - ObjectAttributes: ['Checksum'], - })); + await s3.send( + new GetObjectAttributesCommand({ + Bucket: bucket, + Key: key, + ObjectAttributes: ['Checksum'], + }), + ); assert.fail('Expected NotImplemented error'); } catch (err) { assert.strictEqual(err.name, 'NotImplemented'); @@ -135,42 +149,50 @@ describe('objectGetAttributes', () => { it("shouldn't return ObjectParts for non-MPU objects", async () => { // Requesting only ObjectParts for a non-MPU object break AWS SDK v3 - const data = await s3.send(new GetObjectAttributesCommand({ - Bucket: bucket, - Key: key, - ObjectAttributes: ['ObjectParts', 'ETag'], - })); + const data = await s3.send( + new GetObjectAttributesCommand({ + Bucket: bucket, + Key: key, + ObjectAttributes: ['ObjectParts', 'ETag'], + }), + ); assert.strictEqual(data.ObjectParts, undefined, "ObjectParts shouldn't be present"); assert.strictEqual(data.ETag, expectedMD5); }); it('should return StorageClass', async () => { - const data = await s3.send(new GetObjectAttributesCommand({ - Bucket: bucket, - Key: key, - ObjectAttributes: ['StorageClass'], - })); + const data = await s3.send( + new GetObjectAttributesCommand({ + Bucket: bucket, + Key: key, + ObjectAttributes: ['StorageClass'], + }), + ); assert.strictEqual(data.StorageClass, 'STANDARD'); }); it('should return ObjectSize', async () => { - const data = await s3.send(new GetObjectAttributesCommand({ - Bucket: bucket, - Key: key, - ObjectAttributes: ['ObjectSize'], - })); + const data = await s3.send( + new GetObjectAttributesCommand({ + Bucket: bucket, + Key: key, + ObjectAttributes: ['ObjectSize'], + }), + ); assert.strictEqual(data.ObjectSize, body.length); }); it('should return LastModified', async () => { - const data = await s3.send(new GetObjectAttributesCommand({ - Bucket: bucket, - Key: key, - ObjectAttributes: ['ETag'], - })); + const data = await s3.send( + new GetObjectAttributesCommand({ + Bucket: bucket, + Key: key, + ObjectAttributes: ['ETag'], + }), + ); assert(data.LastModified, 'LastModified should be present'); assert(data.LastModified instanceof Date, 'LastModified should be a Date'); @@ -193,31 +215,37 @@ describe('Test get object attributes with multipart upload', () => { await s3.send(new CreateBucketCommand({ Bucket: bucket })); - const createResult = await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: mpuKey, - })); + const createResult = await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: mpuKey, + }), + ); const uploadId = createResult.UploadId; const partData = Buffer.alloc(partSize, 'a'); const parts = []; for (let i = 1; i <= partCount; i++) { - const uploadResult = await s3.send(new UploadPartCommand({ - Bucket: bucket, - Key: mpuKey, - PartNumber: i, - UploadId: uploadId, - Body: partData, - })); + const uploadResult = await s3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: mpuKey, + PartNumber: i, + UploadId: uploadId, + Body: partData, + }), + ); parts.push({ PartNumber: i, ETag: uploadResult.ETag }); } - await s3.send(new CompleteMultipartUploadCommand({ - Bucket: bucket, - Key: mpuKey, - UploadId: uploadId, - MultipartUpload: { Parts: parts }, - })); + await s3.send( + new CompleteMultipartUploadCommand({ + Bucket: bucket, + Key: mpuKey, + UploadId: uploadId, + MultipartUpload: { Parts: parts }, + }), + ); }); after(async () => { @@ -226,22 +254,26 @@ describe('Test get object attributes with multipart upload', () => { }); it('should return TotalPartsCount for MPU object', async () => { - const data = await s3.send(new GetObjectAttributesCommand({ - Bucket: bucket, - Key: mpuKey, - ObjectAttributes: ['ObjectParts'], - })); + const data = await s3.send( + new GetObjectAttributesCommand({ + Bucket: bucket, + Key: mpuKey, + ObjectAttributes: ['ObjectParts'], + }), + ); assert(data.ObjectParts, 'ObjectParts should be present'); assert.strictEqual(data.ObjectParts.TotalPartsCount, partCount); }); it('should return TotalPartsCount along with other attributes for MPU object', async () => { - const data = await s3.send(new GetObjectAttributesCommand({ - Bucket: bucket, - Key: mpuKey, - ObjectAttributes: ['ETag', 'ObjectParts', 'ObjectSize', 'StorageClass'], - })); + const data = await s3.send( + new GetObjectAttributesCommand({ + Bucket: bucket, + Key: mpuKey, + ObjectAttributes: ['ETag', 'ObjectParts', 'ObjectSize', 'StorageClass'], + }), + ); assert(data.ETag, 'ETag should be present'); assert(data.ETag.includes(`-${partCount}`), `ETag should indicate MPU with ${partCount} parts`); @@ -273,64 +305,76 @@ describe('objectGetAttributes with user metadata', () => { }); it('should return specific user metadata when requested', async () => { - await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: body, - Metadata: { - 'custom-key': 'custom-value', - 'another-key': 'another-value', - }, - })); - - const response = await s3.send(new GetObjectAttributesExtendedCommand({ - Bucket: bucket, - Key: key, - ObjectAttributes: ['x-amz-meta-custom-key'], - })); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: body, + Metadata: { + 'custom-key': 'custom-value', + 'another-key': 'another-value', + }, + }), + ); + + const response = await s3.send( + new GetObjectAttributesExtendedCommand({ + Bucket: bucket, + Key: key, + ObjectAttributes: ['x-amz-meta-custom-key'], + }), + ); assert.strictEqual(response['x-amz-meta-custom-key'], 'custom-value'); }); it('should return multiple user metadata when requested', async () => { - await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: body, - Metadata: { - foo: 'foo-value', - bar: 'bar-value', - baz: 'baz-value', - }, - })); - - const response = await s3.send(new GetObjectAttributesExtendedCommand({ - Bucket: bucket, - Key: key, - ObjectAttributes: ['x-amz-meta-foo', 'x-amz-meta-bar'], - })); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: body, + Metadata: { + foo: 'foo-value', + bar: 'bar-value', + baz: 'baz-value', + }, + }), + ); + + const response = await s3.send( + new GetObjectAttributesExtendedCommand({ + Bucket: bucket, + Key: key, + ObjectAttributes: ['x-amz-meta-foo', 'x-amz-meta-bar'], + }), + ); assert.strictEqual(response['x-amz-meta-foo'], 'foo-value'); assert.strictEqual(response['x-amz-meta-bar'], 'bar-value'); }); it('should return only all user metadata when x-amz-meta-* is requested', async () => { - await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: body, - Metadata: { - key1: 'value1', - key2: 'value2', - key3: 'value3', - }, - })); - - const response = await s3.send(new GetObjectAttributesExtendedCommand({ - Bucket: bucket, - Key: key, - ObjectAttributes: ['x-amz-meta-*'], - })); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: body, + Metadata: { + key1: 'value1', + key2: 'value2', + key3: 'value3', + }, + }), + ); + + const response = await s3.send( + new GetObjectAttributesExtendedCommand({ + Bucket: bucket, + Key: key, + ObjectAttributes: ['x-amz-meta-*'], + }), + ); assert.strictEqual(response['x-amz-meta-key1'], 'value1'); assert.strictEqual(response['x-amz-meta-key2'], 'value2'); @@ -339,75 +383,91 @@ describe('objectGetAttributes with user metadata', () => { }); it('should return empty response when object has no user metadata and x-amz-meta-* is requested', async () => { - await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: body, - })); - - const response = await s3.send(new GetObjectAttributesExtendedCommand({ - Bucket: bucket, - Key: key, - ObjectAttributes: ['ETag', 'x-amz-meta-*'], - })); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: body, + }), + ); + + const response = await s3.send( + new GetObjectAttributesExtendedCommand({ + Bucket: bucket, + Key: key, + ObjectAttributes: ['ETag', 'x-amz-meta-*'], + }), + ); const metadataKeys = Object.keys(response).filter(k => k.startsWith('x-amz-meta-')); assert.strictEqual(metadataKeys.length, 0); }); it('should return empty response when requested metadata key does not exist', async () => { - await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: body, - Metadata: { - existing: 'value', - }, - })); - - const response = await s3.send(new GetObjectAttributesExtendedCommand({ - Bucket: bucket, - Key: key, - ObjectAttributes: ['ETag', 'x-amz-meta-nonexistent'], - })); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: body, + Metadata: { + existing: 'value', + }, + }), + ); + + const response = await s3.send( + new GetObjectAttributesExtendedCommand({ + Bucket: bucket, + Key: key, + ObjectAttributes: ['ETag', 'x-amz-meta-nonexistent'], + }), + ); assert.strictEqual(response['x-amz-meta-nonexistent'], undefined); }); it('should return empty response when only a non-existing metadata key is requested', async () => { - await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: body, - Metadata: { - existing: 'value', - }, - })); - - const response = await s3.send(new GetObjectAttributesExtendedCommand({ - Bucket: bucket, - Key: key, - ObjectAttributes: ['x-amz-meta-nonexistent'], - })); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: body, + Metadata: { + existing: 'value', + }, + }), + ); + + const response = await s3.send( + new GetObjectAttributesExtendedCommand({ + Bucket: bucket, + Key: key, + ObjectAttributes: ['x-amz-meta-nonexistent'], + }), + ); assert.strictEqual(response['x-amz-meta-nonexistent'], undefined); }); it('should return user metadata along with standard attributes', async () => { - await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: body, - Metadata: { - custom: 'custom-value', - }, - })); - - const response = await s3.send(new GetObjectAttributesExtendedCommand({ - Bucket: bucket, - Key: key, - ObjectAttributes: ['ETag', 'x-amz-meta-custom', 'ObjectSize'], - })); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: body, + Metadata: { + custom: 'custom-value', + }, + }), + ); + + const response = await s3.send( + new GetObjectAttributesExtendedCommand({ + Bucket: bucket, + Key: key, + ObjectAttributes: ['ETag', 'x-amz-meta-custom', 'ObjectSize'], + }), + ); assert.strictEqual(response.ETag, expectedMD5); assert.strictEqual(response.ObjectSize, body.length); @@ -415,22 +475,26 @@ describe('objectGetAttributes with user metadata', () => { }); it('should return all metadata once wildcard is provided', async () => { - await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: body, - Metadata: { - key1: 'value1', - key2: 'value2', - key3: 'value3', - }, - })); - - const response = await s3.send(new GetObjectAttributesExtendedCommand({ - Bucket: bucket, - Key: key, - ObjectAttributes: ['x-amz-meta-*', 'x-amz-meta-key1'], - })); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: body, + Metadata: { + key1: 'value1', + key2: 'value2', + key3: 'value3', + }, + }), + ); + + const response = await s3.send( + new GetObjectAttributesExtendedCommand({ + Bucket: bucket, + Key: key, + ObjectAttributes: ['x-amz-meta-*', 'x-amz-meta-key1'], + }), + ); assert.strictEqual(response['x-amz-meta-key1'], 'value1'); assert.strictEqual(response['x-amz-meta-key2'], 'value2'); @@ -438,42 +502,50 @@ describe('objectGetAttributes with user metadata', () => { }); it('should handle duplicate wildcard requests without duplicating results', async () => { - await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: body, - Metadata: { - key1: 'value1', - key2: 'value2', - }, - })); - - const response = await s3.send(new GetObjectAttributesExtendedCommand({ - Bucket: bucket, - Key: key, - ObjectAttributes: ['x-amz-meta-*', 'x-amz-meta-*'], - })); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: body, + Metadata: { + key1: 'value1', + key2: 'value2', + }, + }), + ); + + const response = await s3.send( + new GetObjectAttributesExtendedCommand({ + Bucket: bucket, + Key: key, + ObjectAttributes: ['x-amz-meta-*', 'x-amz-meta-*'], + }), + ); assert.strictEqual(response['x-amz-meta-key1'], 'value1'); assert.strictEqual(response['x-amz-meta-key2'], 'value2'); }); it('should handle duplicate specific metadata requests without duplicating results', async () => { - await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: body, - Metadata: { - key1: 'value1', - key2: 'value2', - }, - })); - - const response = await s3.send(new GetObjectAttributesExtendedCommand({ - Bucket: bucket, - Key: key, - ObjectAttributes: ['x-amz-meta-key1', 'x-amz-meta-key1'], - })); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: body, + Metadata: { + key1: 'value1', + key2: 'value2', + }, + }), + ); + + const response = await s3.send( + new GetObjectAttributesExtendedCommand({ + Bucket: bucket, + Key: key, + ObjectAttributes: ['x-amz-meta-key1', 'x-amz-meta-key1'], + }), + ); assert.strictEqual(response['x-amz-meta-key1'], 'value1'); assert.strictEqual(response['x-amz-meta-key2'], undefined); diff --git a/tests/functional/aws-node-sdk/test/object/objectHead.js b/tests/functional/aws-node-sdk/test/object/objectHead.js index f376afc885..5cae741741 100644 --- a/tests/functional/aws-node-sdk/test/object/objectHead.js +++ b/tests/functional/aws-node-sdk/test/object/objectHead.js @@ -27,8 +27,7 @@ const objectName = 'someObject'; const partSize = 1024 * 1024 * 5; // 5MB minumum required part size. function checkNoError(err) { - assert.equal(err, null, - `Expected success, got error ${JSON.stringify(err)}`); + assert.equal(err, null, `Expected success, got error ${JSON.stringify(err)}`); } function dateFromNow(diff) { @@ -52,78 +51,84 @@ describe('HEAD object, conditions', () => { before(() => { bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; - return bucketUtil.empty(bucketName).then(() => - bucketUtil.deleteOne(bucketName) - ) - .catch(err => { - if (err.name !== 'NoSuchBucket') { - process.stdout.write(`${err}\n`); - throw err; - } - }) - .then(() => bucketUtil.createOne(bucketName)); + return bucketUtil + .empty(bucketName) + .then(() => bucketUtil.deleteOne(bucketName)) + .catch(err => { + if (err.name !== 'NoSuchBucket') { + process.stdout.write(`${err}\n`); + throw err; + } + }) + .then(() => bucketUtil.createOne(bucketName)); }); function requestHead(fields, cb) { - s3.send(new HeadObjectCommand(Object.assign({ - Bucket: bucketName, - Key: objectName, - }, fields))).then(res => cb(null, res)).catch(cb); + s3.send( + new HeadObjectCommand( + Object.assign( + { + Bucket: bucketName, + Key: objectName, + }, + fields, + ), + ), + ) + .then(res => cb(null, res)) + .catch(cb); } - beforeEach(() => s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName, - Body: 'I am the best content ever', - })) - .then(res => { - etag = res.ETag; - etagTrim = etag.substring(1, etag.length - 1); - return s3.send(new HeadObjectCommand( - { Bucket: bucketName, Key: objectName })); - }).then(res => { - lastModified = res.LastModified; - })); + beforeEach(() => + s3 + .send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + Body: 'I am the best content ever', + }), + ) + .then(res => { + etag = res.ETag; + etagTrim = etag.substring(1, etag.length - 1); + return s3.send(new HeadObjectCommand({ Bucket: bucketName, Key: objectName })); + }) + .then(res => { + lastModified = res.LastModified; + }), + ); afterEach(() => bucketUtil.empty(bucketName)); after(() => bucketUtil.deleteOne(bucketName)); - it('If-Match: returns no error when ETag match, with double quotes ' + - 'around ETag', - done => { - requestHead({ IfMatch: etag }, err => { - checkNoError(err); - done(); - }); + it('If-Match: returns no error when ETag match, with double quotes ' + 'around ETag', done => { + requestHead({ IfMatch: etag }, err => { + checkNoError(err); + done(); }); + }); - it('If-Match: returns no error when one of ETags match, with double ' + - 'quotes around ETag', - done => { - requestHead({ IfMatch: `non-matching,${etag}` }, err => { - checkNoError(err); - done(); - }); + it('If-Match: returns no error when one of ETags match, with double ' + 'quotes around ETag', done => { + requestHead({ IfMatch: `non-matching,${etag}` }, err => { + checkNoError(err); + done(); }); + }); - it('If-Match: returns no error when ETag match, without double ' + - 'quotes around ETag', - done => { - requestHead({ IfMatch: etagTrim }, err => { - checkNoError(err); - done(); - }); + it('If-Match: returns no error when ETag match, without double ' + 'quotes around ETag', done => { + requestHead({ IfMatch: etagTrim }, err => { + checkNoError(err); + done(); }); + }); - it('If-Match: returns no error when one of ETags match, without ' + - 'double quotes around ETag', - done => { - requestHead({ IfMatch: `non-matching,${etagTrim}` }, err => { - checkNoError(err); - done(); - }); + it('If-Match: returns no error when one of ETags match, without ' + 'double quotes around ETag', done => { + requestHead({ IfMatch: `non-matching,${etagTrim}` }, err => { + checkNoError(err); + done(); }); + }); it('If-Match: returns no error when ETag match with *', done => { requestHead({ IfMatch: '*' }, err => { @@ -132,13 +137,12 @@ describe('HEAD object, conditions', () => { }); }); - it('If-Match: returns PreconditionFailed when ETag does not match', - done => { - requestHead({ IfMatch: 'non-matching ETag' }, err => { - assert.equal(err.$metadata.httpStatusCode, 412); - done(); - }); + it('If-Match: returns PreconditionFailed when ETag does not match', done => { + requestHead({ IfMatch: 'non-matching ETag' }, err => { + assert.equal(err.$metadata.httpStatusCode, 412); + done(); }); + }); it('If-None-Match: returns no error when ETag does not match', done => { requestHead({ IfNoneMatch: 'non-matching' }, err => { @@ -147,282 +151,320 @@ describe('HEAD object, conditions', () => { }); }); - it('If-None-Match: returns no error when all ETags do not match', - done => { - requestHead({ + it('If-None-Match: returns no error when all ETags do not match', done => { + requestHead( + { IfNoneMatch: 'non-matching,non-matching-either', - }, err => { + }, + err => { checkNoError(err); done(); - }); - }); + }, + ); + }); - it('If-None-Match: returns NotModified when ETag match, with double ' + - 'quotes around ETag', - done => { - requestHead({ IfNoneMatch: etag }, err => { - assert.equal(err.$metadata.httpStatusCode, 304); - done(); - }); + it('If-None-Match: returns NotModified when ETag match, with double ' + 'quotes around ETag', done => { + requestHead({ IfNoneMatch: etag }, err => { + assert.equal(err.$metadata.httpStatusCode, 304); + done(); }); + }); - it('If-None-Match: returns NotModified when one of ETags match, with ' + - 'double quotes around ETag', - done => { - requestHead({ + it('If-None-Match: returns NotModified when one of ETags match, with ' + 'double quotes around ETag', done => { + requestHead( + { IfNoneMatch: `non-matching,${etag}`, - }, err => { - assert.equal(err.$metadata.httpStatusCode, 304); - done(); - }); - }); - - it('If-None-Match: returns NotModified when ETag match, without ' + - 'double quotes around ETag', - done => { - requestHead({ IfNoneMatch: etagTrim }, err => { + }, + err => { assert.equal(err.$metadata.httpStatusCode, 304); done(); - }); - }); + }, + ); + }); - it('If-None-Match: returns NotModified when one of ETags match, ' + - 'without double quotes around ETag', - done => { - requestHead({ - IfNoneMatch: `non-matching,${etagTrim}`, - }, err => { - assert.equal(err.$metadata.httpStatusCode, 304); - done(); - }); + it('If-None-Match: returns NotModified when ETag match, without ' + 'double quotes around ETag', done => { + requestHead({ IfNoneMatch: etagTrim }, err => { + assert.equal(err.$metadata.httpStatusCode, 304); + done(); }); + }); - it('If-Modified-Since: returns no error if Last modified date is ' + - 'greater', + it( + 'If-None-Match: returns NotModified when one of ETags match, ' + 'without double quotes around ETag', done => { - requestHead({ IfModifiedSince: dateFromNow(-1) }, + requestHead( + { + IfNoneMatch: `non-matching,${etagTrim}`, + }, err => { - checkNoError(err); + assert.equal(err.$metadata.httpStatusCode, 304); done(); - }); + }, + ); + }, + ); + + it('If-Modified-Since: returns no error if Last modified date is ' + 'greater', done => { + requestHead({ IfModifiedSince: dateFromNow(-1) }, err => { + checkNoError(err); + done(); }); + }); // Skipping this test, because real AWS does not provide error as // expected - it.skip('If-Modified-Since: returns NotModified if Last modified ' + - 'date is lesser', - done => { - requestHead({ IfModifiedSince: dateFromNow(1) }, - err => { - checkError(err, errorInstances.NotModified.code); - done(); - }); + it.skip('If-Modified-Since: returns NotModified if Last modified ' + 'date is lesser', done => { + requestHead({ IfModifiedSince: dateFromNow(1) }, err => { + checkError(err, errorInstances.NotModified.code); + done(); }); + }); - it('If-Modified-Since: returns NotModified if Last modified ' + - 'date is equal', - done => { - requestHead({ IfModifiedSince: dateConvert(lastModified) }, - err => { - assert.equal(err.$metadata.httpStatusCode, 304); - done(); - }); + it('If-Modified-Since: returns NotModified if Last modified ' + 'date is equal', done => { + requestHead({ IfModifiedSince: dateConvert(lastModified) }, err => { + assert.equal(err.$metadata.httpStatusCode, 304); + done(); }); + }); - it('If-Unmodified-Since: returns no error when lastModified date is ' + - 'greater', - done => { - requestHead({ IfUnmodifiedSince: dateFromNow(1) }, err => { - checkNoError(err); - done(); - }); + it('If-Unmodified-Since: returns no error when lastModified date is ' + 'greater', done => { + requestHead({ IfUnmodifiedSince: dateFromNow(1) }, err => { + checkNoError(err); + done(); }); + }); - it('If-Unmodified-Since: returns no error when lastModified ' + - 'date is equal', - done => { - requestHead({ IfUnmodifiedSince: dateConvert(lastModified) }, - err => { - checkNoError(err); - done(); - }); + it('If-Unmodified-Since: returns no error when lastModified ' + 'date is equal', done => { + requestHead({ IfUnmodifiedSince: dateConvert(lastModified) }, err => { + checkNoError(err); + done(); }); + }); - it('If-Unmodified-Since: returns PreconditionFailed when ' + - 'lastModified date is lesser', - done => { - requestHead({ IfUnmodifiedSince: dateFromNow(-1) }, err => { - assert.equal(err.$metadata.httpStatusCode, 412); - done(); - }); + it('If-Unmodified-Since: returns PreconditionFailed when ' + 'lastModified date is lesser', done => { + requestHead({ IfUnmodifiedSince: dateFromNow(-1) }, err => { + assert.equal(err.$metadata.httpStatusCode, 412); + done(); }); + }); - it('If-Match & If-Unmodified-Since: returns no error when match Etag ' + - 'and lastModified is greater', + it( + 'If-Match & If-Unmodified-Since: returns no error when match Etag ' + 'and lastModified is greater', done => { - requestHead({ + requestHead( + { + IfMatch: etagTrim, + IfUnmodifiedSince: dateFromNow(-1), + }, + err => { + checkNoError(err); + done(); + }, + ); + }, + ); + + it('If-Match match & If-Unmodified-Since match', done => { + requestHead( + { IfMatch: etagTrim, - IfUnmodifiedSince: dateFromNow(-1), - }, err => { + IfUnmodifiedSince: dateFromNow(1), + }, + err => { checkNoError(err); done(); - }); - }); - - it('If-Match match & If-Unmodified-Since match', done => { - requestHead({ - IfMatch: etagTrim, - IfUnmodifiedSince: dateFromNow(1), - }, err => { - checkNoError(err); - done(); - }); + }, + ); }); it('If-Match not match & If-Unmodified-Since not match', done => { - requestHead({ - IfMatch: 'non-matching', - IfUnmodifiedSince: dateFromNow(-1), - }, err => { - assert.equal(err.$metadata.httpStatusCode, 412); - done(); - }); + requestHead( + { + IfMatch: 'non-matching', + IfUnmodifiedSince: dateFromNow(-1), + }, + err => { + assert.equal(err.$metadata.httpStatusCode, 412); + done(); + }, + ); }); it('If-Match not match & If-Unmodified-Since match', done => { - requestHead({ - IfMatch: 'non-matching', - IfUnmodifiedSince: dateFromNow(1), - }, err => { - assert.equal(err.$metadata.httpStatusCode, 412); - done(); - }); + requestHead( + { + IfMatch: 'non-matching', + IfUnmodifiedSince: dateFromNow(1), + }, + err => { + assert.equal(err.$metadata.httpStatusCode, 412); + done(); + }, + ); }); // Skipping this test, because real AWS does not provide error as // expected it.skip('If-Match match & If-Modified-Since not match', done => { - requestHead({ - IfMatch: etagTrim, - IfModifiedSince: dateFromNow(1), - }, err => { - checkNoError(err); - done(); - }); + requestHead( + { + IfMatch: etagTrim, + IfModifiedSince: dateFromNow(1), + }, + err => { + checkNoError(err); + done(); + }, + ); }); it('If-Match match & If-Modified-Since match', done => { - requestHead({ - IfMatch: etagTrim, - IfModifiedSince: dateFromNow(-1), - }, err => { - checkNoError(err); - done(); - }); + requestHead( + { + IfMatch: etagTrim, + IfModifiedSince: dateFromNow(-1), + }, + err => { + checkNoError(err); + done(); + }, + ); }); it('If-Match not match & If-Modified-Since not match', done => { - requestHead({ - IfMatch: 'non-matching', - IfModifiedSince: dateFromNow(1), - }, err => { - assert.equal(err.$metadata.httpStatusCode, 412); - done(); - }); + requestHead( + { + IfMatch: 'non-matching', + IfModifiedSince: dateFromNow(1), + }, + err => { + assert.equal(err.$metadata.httpStatusCode, 412); + done(); + }, + ); }); it('If-Match not match & If-Modified-Since match', done => { - requestHead({ - IfMatch: 'non-matching', - IfModifiedSince: dateFromNow(-1), - }, err => { - assert.equal(err.$metadata.httpStatusCode, 412); - done(); - }); + requestHead( + { + IfMatch: 'non-matching', + IfModifiedSince: dateFromNow(-1), + }, + err => { + assert.equal(err.$metadata.httpStatusCode, 412); + done(); + }, + ); }); - it('If-None-Match & If-Modified-Since: returns NotModified when Etag ' + - 'does not match and lastModified is greater', + it( + 'If-None-Match & If-Modified-Since: returns NotModified when Etag ' + + 'does not match and lastModified is greater', done => { - requestHead({ + requestHead( + { + IfNoneMatch: etagTrim, + IfModifiedSince: dateFromNow(-1), + }, + err => { + assert.equal(err.$metadata.httpStatusCode, 304); + done(); + }, + ); + }, + ); + + it('If-None-Match not match & If-Modified-Since not match', done => { + requestHead( + { IfNoneMatch: etagTrim, - IfModifiedSince: dateFromNow(-1), - }, err => { + IfModifiedSince: dateFromNow(1), + }, + err => { assert.equal(err.$metadata.httpStatusCode, 304); done(); - }); - }); - - it('If-None-Match not match & If-Modified-Since not match', done => { - requestHead({ - IfNoneMatch: etagTrim, - IfModifiedSince: dateFromNow(1), - }, err => { - assert.equal(err.$metadata.httpStatusCode, 304); - done(); - }); + }, + ); }); it('If-None-Match match & If-Modified-Since match', done => { - requestHead({ - IfNoneMatch: 'non-matching', - IfModifiedSince: dateFromNow(-1), - }, err => { - checkNoError(err); - done(); - }); + requestHead( + { + IfNoneMatch: 'non-matching', + IfModifiedSince: dateFromNow(-1), + }, + err => { + checkNoError(err); + done(); + }, + ); }); // Skipping this test, because real AWS does not provide error as // expected it.skip('If-None-Match match & If-Modified-Since not match', done => { - requestHead({ - IfNoneMatch: 'non-matching', - IfModifiedSince: dateFromNow(1), - }, err => { - assert.equal(err.$metadata.httpStatusCode, 304); - done(); - }); + requestHead( + { + IfNoneMatch: 'non-matching', + IfModifiedSince: dateFromNow(1), + }, + err => { + assert.equal(err.$metadata.httpStatusCode, 304); + done(); + }, + ); }); it('If-None-Match match & If-Unmodified-Since match', done => { - requestHead({ - IfNoneMatch: 'non-matching', - IfUnmodifiedSince: dateFromNow(1), - }, err => { - checkNoError(err); - done(); - }); + requestHead( + { + IfNoneMatch: 'non-matching', + IfUnmodifiedSince: dateFromNow(1), + }, + err => { + checkNoError(err); + done(); + }, + ); }); it('If-None-Match match & If-Unmodified-Since not match', done => { - requestHead({ - IfNoneMatch: 'non-matching', - IfUnmodifiedSince: dateFromNow(-1), - }, err => { - assert.equal(err.$metadata.httpStatusCode, 412); - done(); - }); + requestHead( + { + IfNoneMatch: 'non-matching', + IfUnmodifiedSince: dateFromNow(-1), + }, + err => { + assert.equal(err.$metadata.httpStatusCode, 412); + done(); + }, + ); }); it('If-None-Match not match & If-Unmodified-Since match', done => { - requestHead({ - IfNoneMatch: etagTrim, - IfUnmodifiedSince: dateFromNow(1), - }, err => { - assert.equal(err.$metadata.httpStatusCode, 304); - done(); - }); + requestHead( + { + IfNoneMatch: etagTrim, + IfUnmodifiedSince: dateFromNow(1), + }, + err => { + assert.equal(err.$metadata.httpStatusCode, 304); + done(); + }, + ); }); it('If-None-Match not match & If-Unmodified-Since not match', done => { - requestHead({ - IfNoneMatch: etagTrim, - IfUnmodifiedSince: dateFromNow(-1), - }, err => { - assert.equal(err.$metadata.httpStatusCode, 412); - done(); - }); + requestHead( + { + IfNoneMatch: etagTrim, + IfUnmodifiedSince: dateFromNow(-1), + }, + err => { + assert.equal(err.$metadata.httpStatusCode, 412); + done(); + }, + ); }); it('WebsiteRedirectLocation is set & it appears in response', done => { @@ -438,8 +480,7 @@ describe('HEAD object, conditions', () => { }; s3.send(new PutObjectCommand(redirBktwBody)).then(() => { s3.send(new HeadObjectCommand(redirBkt)).then(data => { - assert.strictEqual(data.WebsiteRedirectLocation, - 'http://google.com'); + assert.strictEqual(data.WebsiteRedirectLocation, 'http://google.com'); done(); }); }); @@ -467,78 +508,106 @@ describe('HEAD object, conditions', () => { it('WebsiteRedirectLocation is not set & is absent', done => { requestHead({}, (err, data) => { checkNoError(err); - assert.strictEqual('WebsiteRedirectLocation' in data, - false, 'WebsiteRedirectLocation header is present.'); + assert.strictEqual( + 'WebsiteRedirectLocation' in data, + false, + 'WebsiteRedirectLocation header is present.', + ); done(); }); }); - it('PartNumber is set & PartsCount is absent because object is not ' + - 'multipart', done => { + it('PartNumber is set & PartsCount is absent because object is not ' + 'multipart', done => { requestHead({ PartNumber: 1 }, (err, data) => { assert.ifError(err); - assert.strictEqual('PartsCount' in data, false, - 'PartsCount header is present.'); + assert.strictEqual('PartsCount' in data, false, 'PartsCount header is present.'); done(); }); }); - it('PartNumber is set & PartsCount appears in response for ' + - 'multipart object', done => { + it('PartNumber is set & PartsCount appears in response for ' + 'multipart object', done => { const mpuKey = 'mpukey'; - async.waterfall([ - next => s3.send(new CreateMultipartUploadCommand({ - Bucket: bucketName, - Key: mpuKey, - })).then(data => next(null, data)).catch(next), - (data, next) => { - const uploadId = data.UploadId; - s3.send(new UploadPartCommand({ - Bucket: bucketName, - Key: mpuKey, - UploadId: uploadId, - PartNumber: 1, - Body: Buffer.alloc(partSize).fill('a'), - })).then(data => next(null, uploadId, data.ETag)).catch(next); - }, - (uploadId, etagOne, next) => s3.send(new UploadPartCommand({ - Bucket: bucketName, - Key: mpuKey, - UploadId: uploadId, - PartNumber: 2, - Body: Buffer.alloc(partSize).fill('z'), - })).then(data => next(null, uploadId, etagOne, data.ETag)).catch(next), - (uploadId, etagOne, etagTwo, next) => - s3.send(new CompleteMultipartUploadCommand({ - Bucket: bucketName, - Key: mpuKey, - UploadId: uploadId, - MultipartUpload: { - Parts: [{ - PartNumber: 1, - ETag: etagOne, - }, { - PartNumber: 2, - ETag: etagTwo, - }], + async.waterfall( + [ + next => + s3 + .send( + new CreateMultipartUploadCommand({ + Bucket: bucketName, + Key: mpuKey, + }), + ) + .then(data => next(null, data)) + .catch(next), + (data, next) => { + const uploadId = data.UploadId; + s3.send( + new UploadPartCommand({ + Bucket: bucketName, + Key: mpuKey, + UploadId: uploadId, + PartNumber: 1, + Body: Buffer.alloc(partSize).fill('a'), + }), + ) + .then(data => next(null, uploadId, data.ETag)) + .catch(next); }, - })).then(data => next(null, data)).catch(next), - ], err => { - assert.ifError(err); - s3.send(new HeadObjectCommand({ - Bucket: bucketName, - Key: mpuKey, - PartNumber: 1, - })).then(data => { - assert.strictEqual(data.PartsCount, 2); - done(); - }); - }); + (uploadId, etagOne, next) => + s3 + .send( + new UploadPartCommand({ + Bucket: bucketName, + Key: mpuKey, + UploadId: uploadId, + PartNumber: 2, + Body: Buffer.alloc(partSize).fill('z'), + }), + ) + .then(data => next(null, uploadId, etagOne, data.ETag)) + .catch(next), + (uploadId, etagOne, etagTwo, next) => + s3 + .send( + new CompleteMultipartUploadCommand({ + Bucket: bucketName, + Key: mpuKey, + UploadId: uploadId, + MultipartUpload: { + Parts: [ + { + PartNumber: 1, + ETag: etagOne, + }, + { + PartNumber: 2, + ETag: etagTwo, + }, + ], + }, + }), + ) + .then(data => next(null, data)) + .catch(next), + ], + err => { + assert.ifError(err); + s3.send( + new HeadObjectCommand({ + Bucket: bucketName, + Key: mpuKey, + PartNumber: 1, + }), + ).then(data => { + assert.strictEqual(data.PartsCount, 2); + done(); + }); + }, + ); }); }); }); - describe('HEAD object with object lock', () => { withV4(sigCfg => { const bucketUtil = new BucketUtility('default', sigCfg); @@ -558,13 +627,15 @@ describe('HEAD object with object lock', () => { ObjectLockMode: mockMode, ObjectLockLegalHoldStatus: 'ON', }; - await s3.send(new CreateBucketCommand({ - Bucket: bucket, - ObjectLockEnabledForBucket: true, - })); + await s3.send( + new CreateBucketCommand({ + Bucket: bucket, + ObjectLockEnabledForBucket: true, + }), + ); await s3.send(new PutObjectCommand(params)); const res = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key })); - + versionId = res.VersionId; }); @@ -588,7 +659,7 @@ describe('HEAD object with object lock', () => { it('should return object lock headers if set on the object', async () => { const res = await s3.send(new HeadObjectCommand({ Bucket: bucket, Key: key })); assert.strictEqual(res.ObjectLockMode, mockMode); - const responseDate= formatDate(res.ObjectLockRetainUntilDate); + const responseDate = formatDate(res.ObjectLockRetainUntilDate); const expectedDate = formatDate(mockDate); assert.strictEqual(responseDate, expectedDate); assert.strictEqual(res.ObjectLockLegalHoldStatus, 'ON'); diff --git a/tests/functional/aws-node-sdk/test/object/objectHead_compatibleHeaders.js b/tests/functional/aws-node-sdk/test/object/objectHead_compatibleHeaders.js index b4603858aa..075114d0e9 100644 --- a/tests/functional/aws-node-sdk/test/object/objectHead_compatibleHeaders.js +++ b/tests/functional/aws-node-sdk/test/object/objectHead_compatibleHeaders.js @@ -2,74 +2,72 @@ const assert = require('assert'); const withV4 = require('../support/withV4'); const BucketUtility = require('../../lib/utility/bucket-util'); -const { PutObjectCommand , HeadObjectCommand } = require('@aws-sdk/client-s3'); +const { PutObjectCommand, HeadObjectCommand } = require('@aws-sdk/client-s3'); const bucketName = 'objectheadtestheaders'; const objectName = 'someObject'; -describe('HEAD object, compatibility headers [Cache-Control, ' + - 'Content-Disposition, Content-Encoding, Expires]', () => { - withV4(sigCfg => { - let bucketUtil; - let s3; - const cacheControl = 'max-age=86400'; - const contentDisposition = 'attachment; filename="fname.ext";'; - const contentEncoding = 'gzip,aws-chunked'; - // AWS Node SDK requires Date object, ISO-8601 string, or - // a UNIX timestamp for Expires header - const expires = new Date(); +describe( + 'HEAD object, compatibility headers [Cache-Control, ' + 'Content-Disposition, Content-Encoding, Expires]', + () => { + withV4(sigCfg => { + let bucketUtil; + let s3; + const cacheControl = 'max-age=86400'; + const contentDisposition = 'attachment; filename="fname.ext";'; + const contentEncoding = 'gzip,aws-chunked'; + // AWS Node SDK requires Date object, ISO-8601 string, or + // a UNIX timestamp for Expires header + const expires = new Date(); - before(() => { - bucketUtil = new BucketUtility('default', sigCfg); - s3 = bucketUtil.s3; - return bucketUtil.empty(bucketName).then(() => - bucketUtil.deleteOne(bucketName) - ) - .catch(err => { - if (err.name !== 'NoSuchBucket') { - process.stdout.write(`${err}\n`); - throw err; - } - }) - .then(() => bucketUtil.createOne(bucketName)) - .then(() => { - const params = { - Bucket: bucketName, - Key: objectName, - CacheControl: cacheControl, - ContentDisposition: contentDisposition, - ContentEncoding: contentEncoding, - Expires: expires, - }; - return s3.send(new PutObjectCommand(params)); + before(() => { + bucketUtil = new BucketUtility('default', sigCfg); + s3 = bucketUtil.s3; + return bucketUtil + .empty(bucketName) + .then(() => bucketUtil.deleteOne(bucketName)) + .catch(err => { + if (err.name !== 'NoSuchBucket') { + process.stdout.write(`${err}\n`); + throw err; + } + }) + .then(() => bucketUtil.createOne(bucketName)) + .then(() => { + const params = { + Bucket: bucketName, + Key: objectName, + CacheControl: cacheControl, + ContentDisposition: contentDisposition, + ContentEncoding: contentEncoding, + Expires: expires, + }; + return s3.send(new PutObjectCommand(params)); + }); }); - }); - after(async () => { - process.stdout.write('deleting bucket'); - await bucketUtil.empty(bucketName); - await bucketUtil.deleteOne(bucketName); - }); + after(async () => { + process.stdout.write('deleting bucket'); + await bucketUtil.empty(bucketName); + await bucketUtil.deleteOne(bucketName); + }); - it('should return additional headers if specified in objectPUT ' + - 'request', done => { - s3.send(new HeadObjectCommand({ Bucket: bucketName, Key: objectName })) - .then(res => { - assert.strictEqual(res.CacheControl, - cacheControl); - assert.strictEqual(res.ContentDisposition, - contentDisposition); - // Should remove V4 streaming value 'aws-chunked' - // to be compatible with AWS behavior - assert.strictEqual(res.ContentEncoding, - 'gzip,'); - assert.strictEqual(res.Expires.toGMTString(), - expires.toGMTString()); - return done(); - }).catch(err => { - process.stdout.write(`Error on headObject: ${err}\n`); - return done(err); - }); + it('should return additional headers if specified in objectPUT ' + 'request', done => { + s3.send(new HeadObjectCommand({ Bucket: bucketName, Key: objectName })) + .then(res => { + assert.strictEqual(res.CacheControl, cacheControl); + assert.strictEqual(res.ContentDisposition, contentDisposition); + // Should remove V4 streaming value 'aws-chunked' + // to be compatible with AWS behavior + assert.strictEqual(res.ContentEncoding, 'gzip,'); + assert.strictEqual(res.Expires.toGMTString(), expires.toGMTString()); + return done(); + }) + .catch(err => { + process.stdout.write(`Error on headObject: ${err}\n`); + return done(err); + }); + }); }); - }); -}); + }, +); diff --git a/tests/functional/aws-node-sdk/test/object/objectHead_replication.js b/tests/functional/aws-node-sdk/test/object/objectHead_replication.js index 39beb5e388..c5c0dde653 100644 --- a/tests/functional/aws-node-sdk/test/object/objectHead_replication.js +++ b/tests/functional/aws-node-sdk/test/object/objectHead_replication.js @@ -2,14 +2,15 @@ const assert = require('assert'); const withV4 = require('../support/withV4'); const BucketUtility = require('../../lib/utility/bucket-util'); -const { removeAllVersions, versioningEnabled } = - require('../../lib/utility/versioning-util'); -const { PutObjectCommand, +const { removeAllVersions, versioningEnabled } = require('../../lib/utility/versioning-util'); +const { + PutObjectCommand, HeadObjectCommand, - CreateBucketCommand, + CreateBucketCommand, DeleteBucketCommand, - PutBucketVersioningCommand, - PutBucketReplicationCommand } = require('@aws-sdk/client-s3'); + PutBucketVersioningCommand, + PutBucketReplicationCommand, +} = require('@aws-sdk/client-s3'); const sourceBucket = 'source-bucket'; const keyPrefix = 'test-prefix'; @@ -27,10 +28,12 @@ describe("Head object 'ReplicationStatus' value", () => { beforeEach(async () => { await s3.send(new CreateBucketCommand({ Bucket: sourceBucket })); - await s3.send(new PutBucketVersioningCommand({ - Bucket: sourceBucket, - VersioningConfiguration: versioningEnabled, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: sourceBucket, + VersioningConfiguration: versioningEnabled, + }), + ); }); afterEach(done => { @@ -38,40 +41,43 @@ describe("Head object 'ReplicationStatus' value", () => { if (err) { return done(err); } - return s3.send(new DeleteBucketCommand({ Bucket: sourceBucket })) - .then(() => done()).catch(done); + return s3 + .send(new DeleteBucketCommand({ Bucket: sourceBucket })) + .then(() => done()) + .catch(done); }); }); - it('should be `undefined` when there is no bucket replication config', - async () => await checkHeadObj(`${keyPrefix}-foobar`, undefined)); + it('should be `undefined` when there is no bucket replication config', async () => + await checkHeadObj(`${keyPrefix}-foobar`, undefined)); describe('With bucket replication config', () => { const role = process.env.S3_END_TO_END - ? 'arn:aws:iam::123456789012:role/src-resource,arn:aws:iam::123456789012:role/dest-resource' - : 'arn:aws:iam::123456789012:role/src-resource'; + ? 'arn:aws:iam::123456789012:role/src-resource,arn:aws:iam::123456789012:role/dest-resource' + : 'arn:aws:iam::123456789012:role/src-resource'; beforeEach(async () => { - await s3.send(new PutBucketReplicationCommand({ - Bucket: sourceBucket, - ReplicationConfiguration: { - Role: role, - Rules: [ - { - Destination: { StorageClass: 'us-east-2', - Bucket: 'arn:aws:s3:::dest-bucket' }, - Prefix: keyPrefix, - Status: 'Enabled', - }, - ], - }, - })); + await s3.send( + new PutBucketReplicationCommand({ + Bucket: sourceBucket, + ReplicationConfiguration: { + Role: role, + Rules: [ + { + Destination: { StorageClass: 'us-east-2', Bucket: 'arn:aws:s3:::dest-bucket' }, + Prefix: keyPrefix, + Status: 'Enabled', + }, + ], + }, + }), + ); }); - it("should be 'PENDING' when object key prefix applies", - async () => await checkHeadObj(`${keyPrefix}-foobar`, 'PENDING')); + it("should be 'PENDING' when object key prefix applies", async () => + await checkHeadObj(`${keyPrefix}-foobar`, 'PENDING')); - it('should be `undefined` when object key prefix does not apply', - async () => await checkHeadObj(`foobar-${keyPrefix}`, undefined)); + it('should be `undefined` when object key prefix does not apply', async () => + await checkHeadObj(`foobar-${keyPrefix}`, undefined)); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/object/objectOverwrite.js b/tests/functional/aws-node-sdk/test/object/objectOverwrite.js index 177339898c..7efc12400e 100644 --- a/tests/functional/aws-node-sdk/test/object/objectOverwrite.js +++ b/tests/functional/aws-node-sdk/test/object/objectOverwrite.js @@ -56,7 +56,6 @@ const secondPutMetadata = { secondputagain: 'secondValue', }; - describe('Put object with same key as prior object', () => { withV4(sigCfg => { let bucketUtil; @@ -71,16 +70,20 @@ describe('Put object with same key as prior object', () => { }); beforeEach(async () => { - await s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName, - Body: 'I am the best content ever', - Metadata: firstPutMetadata, - })); - const res = await s3.send(new HeadObjectCommand({ - Bucket: bucketName, - Key: objectName - })); + await s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + Body: 'I am the best content ever', + Metadata: firstPutMetadata, + }), + ); + const res = await s3.send( + new HeadObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); assert.deepStrictEqual(res.Metadata, firstPutMetadata); }); @@ -88,22 +91,25 @@ describe('Put object with same key as prior object', () => { after(async () => await bucketUtil.deleteOne(bucketName)); - it('should overwrite all user metadata and data on overwrite put', - async () => { - await s3.send(new PutObjectCommand({ + it('should overwrite all user metadata and data on overwrite put', async () => { + await s3.send( + new PutObjectCommand({ Bucket: bucketName, Key: objectName, Body: 'Much different', Metadata: secondPutMetadata, - })); - const res = await s3.send(new GetObjectCommand({ - Bucket: bucketName, - Key: objectName - })); - assert.deepStrictEqual(res.Metadata, secondPutMetadata); - const bodyText = await res.Body.transformToString(); - assert.deepStrictEqual(bodyText, 'Much different'); - }); + }), + ); + const res = await s3.send( + new GetObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); + assert.deepStrictEqual(res.Metadata, secondPutMetadata); + const bodyText = await res.Body.transformToString(); + assert.deepStrictEqual(bodyText, 'Much different'); + }); coldStateScenarios.forEach(({ name, transitionInProgress, archiveState }) => { it(`should replace object with cold-state metadata (${name}) in non-versioned bucket`, async () => { @@ -113,12 +119,14 @@ describe('Put object with same key as prior object', () => { await fakeMetadataArchive(bucketName, objectName, undefined, archiveState); } - await s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName, - Body: `overwrite cold state ${name}`, - Metadata: secondPutMetadata, - })); + await s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + Body: `overwrite cold state ${name}`, + Metadata: secondPutMetadata, + }), + ); const currentMD = await getMetadata(bucketName, objectName, undefined); assert.strictEqual(currentMD.archive, undefined); @@ -129,17 +137,21 @@ describe('Put object with same key as prior object', () => { it('should create a new version when replacing archived current object in versioned bucket', async () => { await bucketUtil.empty(bucketName); - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: { Status: 'Enabled' }, - })); - - const firstPutRes = await s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName, - Body: 'versioned first payload', - Metadata: firstPutMetadata, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ); + + const firstPutRes = await s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + Body: 'versioned first payload', + Metadata: firstPutMetadata, + }), + ); assert(firstPutRes.VersionId); await fakeMetadataArchive(bucketName, objectName, undefined, { @@ -148,19 +160,23 @@ describe('Put object with same key as prior object', () => { restoreRequestedDays: 5, }); - const secondPutRes = await s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName, - Body: 'versioned second payload', - Metadata: secondPutMetadata, - })); + const secondPutRes = await s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + Body: 'versioned second payload', + Metadata: secondPutMetadata, + }), + ); assert(secondPutRes.VersionId); assert.notStrictEqual(secondPutRes.VersionId, firstPutRes.VersionId); - const headRes = await s3.send(new HeadObjectCommand({ - Bucket: bucketName, - Key: objectName, - })); + const headRes = await s3.send( + new HeadObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); assert.deepStrictEqual(headRes.Metadata, secondPutMetadata); const currentMD = await getMetadata(bucketName, objectName, undefined); @@ -170,25 +186,33 @@ describe('Put object with same key as prior object', () => { it('should replace archived current null version in version-suspended bucket', async () => { await bucketUtil.empty(bucketName); - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: { Status: 'Enabled' }, - })); - await s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName, - Body: 'enabled-version-payload', - })); - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: { Status: 'Suspended' }, - })); - - await s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName, - Body: 'null-current-before-archive', - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ); + await s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + Body: 'enabled-version-payload', + }), + ); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: { Status: 'Suspended' }, + }), + ); + + await s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + Body: 'null-current-before-archive', + }), + ); await fakeMetadataArchive(bucketName, objectName, undefined, { archiveInfo: { archiveId: 'archive-null-current' }, @@ -196,17 +220,18 @@ describe('Put object with same key as prior object', () => { restoreRequestedDays: 5, }); - await s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName, - Body: 'replace archived null current', - Metadata: secondPutMetadata, - })); + await s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + Body: 'replace archived null current', + Metadata: secondPutMetadata, + }), + ); const currentMD = await getMetadata(bucketName, objectName, undefined); assert.strictEqual(currentMD.archive, undefined); assert.deepStrictEqual(currentMD['x-amz-meta-secondput'], secondPutMetadata.secondput); }); - }); }); diff --git a/tests/functional/aws-node-sdk/test/object/put.js b/tests/functional/aws-node-sdk/test/object/put.js index aa1514c87d..124f217a14 100644 --- a/tests/functional/aws-node-sdk/test/object/put.js +++ b/tests/functional/aws-node-sdk/test/object/put.js @@ -2,7 +2,8 @@ const assert = require('assert'); const fs = require('fs'); const path = require('path'); const util = require('util'); -const { CreateBucketCommand, +const { + CreateBucketCommand, PutObjectCommand, GetObjectAclCommand, GetObjectTaggingCommand, @@ -13,10 +14,8 @@ const BucketUtility = require('../../lib/utility/bucket-util'); const checkError = require('../../lib/utility/checkError'); const provideRawOutput = require('../../lib/utility/provideRawOutput'); const provideRawOutputAsync = util.promisify(provideRawOutput); -const { taggingTests, generateMultipleTagQuery } - = require('../../lib/utility/tagging'); -const genMaxSizeMetaHeaders - = require('../../lib/utility/genMaxSizeMetaHeaders'); +const { taggingTests, generateMultipleTagQuery } = require('../../lib/utility/tagging'); +const genMaxSizeMetaHeaders = require('../../lib/utility/genMaxSizeMetaHeaders'); const changeObjectLock = require('../../../../utilities/objectLock-util'); const object = 'object2putstuffin'; @@ -35,131 +34,144 @@ describe('PUT object', () => { afterEach(() => { process.stdout.write('Emptying bucket'); - return bucketUtil.empty(bucket) - .then(() => { - process.stdout.write('Deleting bucket'); - return bucketUtil.deleteOne(bucket); - }) - .catch(err => { - process.stdout.write('Error in afterEach'); - throw err; - }); + return bucketUtil + .empty(bucket) + .then(() => { + process.stdout.write('Deleting bucket'); + return bucketUtil.deleteOne(bucket); + }) + .catch(err => { + process.stdout.write('Error in afterEach'); + throw err; + }); }); - it('should put an object and set the acl via query param', - async () => { - // Create a temporary file for upload - const tempFile = path.join(__dirname, 'temp-upload-file.txt'); - fs.writeFileSync(tempFile, 'test content for upload'); - const params = { Bucket: bucket, Key: 'key', - ACL: 'public-read', StorageClass: 'STANDARD' }; - - const command = new PutObjectCommand(params); - const url = await getSignedUrl(s3, command); - const { httpCode } = await provideRawOutputAsync(['-verbose', '-X', 'PUT', url, - '--upload-file', tempFile]); - fs.unlinkSync(tempFile); - assert.strictEqual(httpCode, '200 OK'); - const result = await s3.send(new GetObjectAclCommand({ Bucket: bucket, Key: 'key' })); - assert.deepStrictEqual(result.Grants[1], { Grantee: - { Type: 'Group', URI: - 'http://acs.amazonaws.com/groups/global/AllUsers', - }, Permission: 'READ' }); + it('should put an object and set the acl via query param', async () => { + // Create a temporary file for upload + const tempFile = path.join(__dirname, 'temp-upload-file.txt'); + fs.writeFileSync(tempFile, 'test content for upload'); + const params = { Bucket: bucket, Key: 'key', ACL: 'public-read', StorageClass: 'STANDARD' }; + + const command = new PutObjectCommand(params); + const url = await getSignedUrl(s3, command); + const { httpCode } = await provideRawOutputAsync(['-verbose', '-X', 'PUT', url, '--upload-file', tempFile]); + fs.unlinkSync(tempFile); + assert.strictEqual(httpCode, '200 OK'); + const result = await s3.send(new GetObjectAclCommand({ Bucket: bucket, Key: 'key' })); + assert.deepStrictEqual(result.Grants[1], { + Grantee: { Type: 'Group', URI: 'http://acs.amazonaws.com/groups/global/AllUsers' }, + Permission: 'READ', }); + }); - it('should put an object with key slash', - done => { - const params = { Bucket: bucket, Key: '/' }; - s3.send(new PutObjectCommand(params)).then(() => { + it('should put an object with key slash', done => { + const params = { Bucket: bucket, Key: '/' }; + s3.send(new PutObjectCommand(params)) + .then(() => { done(); - }).catch(err => { - assert.equal(err, null, 'Expected success, ' + - `got error ${JSON.stringify(err)}`); + }) + .catch(err => { + assert.equal(err, null, 'Expected success, ' + `got error ${JSON.stringify(err)}`); done(err); }); - }); + }); it('should return KeyTooLong error when key is longer than 915 bytes', done => { const params = { Bucket: bucket, Key: 'a'.repeat(916) }; - s3.send(new PutObjectCommand(params)).then(() => { - assert(false, 'Expected failure but got success'); - }).catch(err => { - assert(err, 'Expected error but did not find one'); - assert.strictEqual(err.name, 'KeyTooLong'); - assert.match(err.message, /915/); - done(); - }); + s3.send(new PutObjectCommand(params)) + .then(() => { + assert(false, 'Expected failure but got success'); + }) + .catch(err => { + assert(err, 'Expected error but did not find one'); + assert.strictEqual(err.name, 'KeyTooLong'); + assert.match(err.message, /915/); + done(); + }); }); - it('should return error if putting object w/ > 2KB user-defined md', - done => { - const metadata = genMaxSizeMetaHeaders(); - const params = { Bucket: bucket, Key: '/', Metadata: metadata }; - s3.send(new PutObjectCommand(params)).then(() => { + it('should return error if putting object w/ > 2KB user-defined md', done => { + const metadata = genMaxSizeMetaHeaders(); + const params = { Bucket: bucket, Key: '/', Metadata: metadata }; + s3.send(new PutObjectCommand(params)) + .then(() => { // add one more byte to be over the limit metadata.header0 = `${metadata.header0}${'0'}`; - s3.send(new PutObjectCommand(params)).then(() => { + s3.send(new PutObjectCommand(params)) + .then(() => { + assert(false, 'Expected failure but got success'); + }) + .catch(err => { + assert(err, 'Expected err but did not find one'); + assert.strictEqual(err.name, 'MetadataTooLarge'); + assert.strictEqual(err.$metadata.httpStatusCode, 400); + done(); + }); + }) + .catch(err => { + assert.equal(err, null, 'Expected success, ' + `got error ${JSON.stringify(err)}`); + done(err); + }); + }); + + it( + 'should return InvalidRequest error if putting object with ' + + 'object lock retention date and mode when object lock is not ' + + 'enabled on the bucket', + done => { + const date = new Date(2050, 10, 10); + const params = { + Bucket: bucket, + Key: 'key', + ObjectLockRetainUntilDate: date, + ObjectLockMode: 'GOVERNANCE', + }; + s3.send(new PutObjectCommand(params)) + .then(() => { assert(false, 'Expected failure but got success'); - }).catch(err => { - assert(err, 'Expected err but did not find one'); - assert.strictEqual(err.name, 'MetadataTooLarge'); + }) + .catch(err => { + const expectedErrMessage = 'Bucket is missing ObjectLockConfiguration'; + assert.strictEqual(err.name, 'InvalidRequest'); assert.strictEqual(err.$metadata.httpStatusCode, 400); + assert(err.toString().includes(expectedErrMessage)); done(); }); - }).catch(err => { - assert.equal(err, null, 'Expected success, ' + - `got error ${JSON.stringify(err)}`); - done(err); - }); - }); + }, + ); - it('should return InvalidRequest error if putting object with ' + - 'object lock retention date and mode when object lock is not ' + - 'enabled on the bucket', done => { - const date = new Date(2050, 10, 10); - const params = { - Bucket: bucket, - Key: 'key', - ObjectLockRetainUntilDate: date, - ObjectLockMode: 'GOVERNANCE', - }; - s3.send(new PutObjectCommand(params)).then(() => { - assert(false, 'Expected failure but got success'); - }).catch(err => { - const expectedErrMessage - = 'Bucket is missing ObjectLockConfiguration'; - assert.strictEqual(err.name, 'InvalidRequest'); - assert.strictEqual(err.$metadata.httpStatusCode, 400); - assert(err.toString().includes(expectedErrMessage)); - done(); - }); - }); - - it('should return Not Implemented error for obj. encryption using ' + - 'customer-provided encryption keys', done => { - const params = { Bucket: bucket, Key: 'key', - SSECustomerAlgorithm: 'AES256' }; - s3.send(new PutObjectCommand(params)).then(() => { - assert(false, 'Expected failure but got success'); - }).catch(err => { - assert.strictEqual(err.name, 'NotImplemented'); - done(); - }); - }); + it( + 'should return Not Implemented error for obj. encryption using ' + 'customer-provided encryption keys', + done => { + const params = { Bucket: bucket, Key: 'key', SSECustomerAlgorithm: 'AES256' }; + s3.send(new PutObjectCommand(params)) + .then(() => { + assert(false, 'Expected failure but got success'); + }) + .catch(err => { + assert.strictEqual(err.name, 'NotImplemented'); + done(); + }); + }, + ); - it('should return InvalidRedirectLocation if putting object ' + - 'with x-amz-website-redirect-location header that does not start ' + - 'with \'http://\', \'https://\' or \'/\'', done => { - const params = { Bucket: bucket, Key: 'key', - WebsiteRedirectLocation: 'google.com' }; - s3.send(new PutObjectCommand(params)).then(() => { - assert(false, 'Expected failure but got success'); - }).catch(err => { - assert.strictEqual(err.name, 'InvalidRedirectLocation'); - assert.strictEqual(err.$metadata.httpStatusCode, 400); - done(); - }); - }); + it( + 'should return InvalidRedirectLocation if putting object ' + + 'with x-amz-website-redirect-location header that does not start ' + + "with 'http://', 'https://' or '/'", + done => { + const params = { Bucket: bucket, Key: 'key', WebsiteRedirectLocation: 'google.com' }; + s3.send(new PutObjectCommand(params)) + .then(() => { + assert(false, 'Expected failure but got success'); + }) + .catch(err => { + assert.strictEqual(err.name, 'InvalidRedirectLocation'); + assert.strictEqual(err.$metadata.httpStatusCode, 400); + done(); + }); + }, + ); describe('Put object with tag set', () => { taggingTests.forEach(taggingTest => { @@ -167,153 +179,157 @@ describe('PUT object', () => { const key = encodeURIComponent(taggingTest.tag.key); const value = encodeURIComponent(taggingTest.tag.value); const tagging = `${key}=${value}`; - const params = { Bucket: bucket, Key: object, - Tagging: tagging }; - s3.send(new PutObjectCommand(params)).then(() => - s3.send(new GetObjectTaggingCommand({ Bucket: bucket, - Key: object })).then(data => { - assert.deepStrictEqual(data.TagSet[0], { - Key: taggingTest.tag.key, - Value: taggingTest.tag.value }); - done(); - }).catch(err => { - assert.equal(err, null, 'Expected success, ' + - `got error ${JSON.stringify(err)}`); - done(); - })).catch(err => { - if (taggingTest.error) { - checkError(err, taggingTest.error, 400); + const params = { Bucket: bucket, Key: object, Tagging: tagging }; + s3.send(new PutObjectCommand(params)) + .then(() => + s3 + .send(new GetObjectTaggingCommand({ Bucket: bucket, Key: object })) + .then(data => { + assert.deepStrictEqual(data.TagSet[0], { + Key: taggingTest.tag.key, + Value: taggingTest.tag.value, + }); + done(); + }) + .catch(err => { + assert.equal(err, null, 'Expected success, ' + `got error ${JSON.stringify(err)}`); + done(); + }), + ) + .catch(err => { + if (taggingTest.error) { + checkError(err, taggingTest.error, 400); + return done(); + } + assert.equal(err, null, 'Expected success, ' + `got error ${JSON.stringify(err)}`); return done(); - } - assert.equal(err, null, 'Expected success, ' + - `got error ${JSON.stringify(err)}`); - return done(); + }); }); }); - }); - it('should be able to put object with 10 tags', - done => { + it('should be able to put object with 10 tags', done => { const taggingConfig = generateMultipleTagQuery(10); - s3.send(new PutObjectCommand({ Bucket: bucket, Key: object, - Tagging: taggingConfig })).then(() => { - done(); - }).catch(err => { - assert.equal(err, null, 'Expected success, ' + - `got error ${JSON.stringify(err)}`); - done(); - }); + s3.send(new PutObjectCommand({ Bucket: bucket, Key: object, Tagging: taggingConfig })) + .then(() => { + done(); + }) + .catch(err => { + assert.equal(err, null, 'Expected success, ' + `got error ${JSON.stringify(err)}`); + done(); + }); }); it('should be able to put an empty Tag set', done => { - s3.send(new PutObjectCommand({ Bucket: bucket, Key: object, - Tagging: '', - })).then(() => { - done(); - }).catch(err => { - assert.equal(err, null, 'Expected success, ' + - `got error ${JSON.stringify(err)}`); - done(); - }); + s3.send(new PutObjectCommand({ Bucket: bucket, Key: object, Tagging: '' })) + .then(() => { + done(); + }) + .catch(err => { + assert.equal(err, null, 'Expected success, ' + `got error ${JSON.stringify(err)}`); + done(); + }); }); - it('should be able to put object with empty tags', - done => { - s3.send(new PutObjectCommand({ Bucket: bucket, Key: object, - Tagging: '&&&&&&&&&&&&&&&&&key1=value1' })).then(() => { - done(); - }).catch(err => { - assert.equal(err, null, 'Expected success, ' + - `got error ${JSON.stringify(err)}`); - done(); - }); + it('should be able to put object with empty tags', done => { + s3.send(new PutObjectCommand({ Bucket: bucket, Key: object, Tagging: '&&&&&&&&&&&&&&&&&key1=value1' })) + .then(() => { + done(); + }) + .catch(err => { + assert.equal(err, null, 'Expected success, ' + `got error ${JSON.stringify(err)}`); + done(); + }); }); it('should allow putting 50 tags', done => { const taggingConfig = generateMultipleTagQuery(50); - s3.send(new PutObjectCommand({ Bucket: bucket, Key: object, - Tagging: taggingConfig })).then(() => { - done(); - }).catch(err => { - assert.equal(err, null, 'Expected success, ' + - `got error ${JSON.stringify(err)}`); - done(); - }); + s3.send(new PutObjectCommand({ Bucket: bucket, Key: object, Tagging: taggingConfig })) + .then(() => { + done(); + }) + .catch(err => { + assert.equal(err, null, 'Expected success, ' + `got error ${JSON.stringify(err)}`); + done(); + }); }); - it('should return BadRequest if putting more that 50 tags', - done => { + it('should return BadRequest if putting more that 50 tags', done => { const taggingConfig = generateMultipleTagQuery(51); - s3.send(new PutObjectCommand({ Bucket: bucket, Key: object, - Tagging: taggingConfig })).then(() => { - assert(false, 'Expected failure but got success'); - }).catch(err => { - checkError(err, 'BadRequest', 400); - done(); - }); + s3.send(new PutObjectCommand({ Bucket: bucket, Key: object, Tagging: taggingConfig })) + .then(() => { + assert(false, 'Expected failure but got success'); + }) + .catch(err => { + checkError(err, 'BadRequest', 400); + done(); + }); }); - it('should return InvalidArgument if using the same key twice', - done => { - s3.send(new PutObjectCommand({ Bucket: bucket, Key: object, - Tagging: 'key1=value1&key1=value2' })).then(() => { - assert(false, 'Expected failure but got success'); - }).catch(err => { - checkError(err, 'InvalidArgument', 400); - done(); - }); + it('should return InvalidArgument if using the same key twice', done => { + s3.send(new PutObjectCommand({ Bucket: bucket, Key: object, Tagging: 'key1=value1&key1=value2' })) + .then(() => { + assert(false, 'Expected failure but got success'); + }) + .catch(err => { + checkError(err, 'InvalidArgument', 400); + done(); + }); }); - it('should return InvalidArgument if using the same key twice ' + - 'and empty tags', done => { - s3.send(new PutObjectCommand({ Bucket: bucket, Key: object, - Tagging: '&&&&&&&&&&&&&&&&&key1=value1&key1=value2' })).then(() => { - assert(false, 'Expected failure but got success'); - - }).catch(err => { - checkError(err, 'InvalidArgument', 400); - done(); - }); + it('should return InvalidArgument if using the same key twice ' + 'and empty tags', done => { + s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: object, + Tagging: '&&&&&&&&&&&&&&&&&key1=value1&key1=value2', + }), + ) + .then(() => { + assert(false, 'Expected failure but got success'); + }) + .catch(err => { + checkError(err, 'InvalidArgument', 400); + done(); + }); }); it('should return InvalidArgument if tag with no key', done => { - s3.send(new PutObjectCommand({ Bucket: bucket, Key: object, - Tagging: '=value1', - })).then(() => { - assert(false, 'Expected failure but got success'); - }).catch(err => { - checkError(err, 'InvalidArgument', 400); - done(); - }); + s3.send(new PutObjectCommand({ Bucket: bucket, Key: object, Tagging: '=value1' })) + .then(() => { + assert(false, 'Expected failure but got success'); + }) + .catch(err => { + checkError(err, 'InvalidArgument', 400); + done(); + }); }); - it('should return InvalidArgument putting object with ' + - 'bad encoded tags', done => { - s3.send(new PutObjectCommand({ Bucket: bucket, Key: object, Tagging: - 'key1==value1' })).then(() => { - assert(false, 'Expected failure but got success'); - }).catch(err => { - checkError(err, 'InvalidArgument', 400); - done(); - }); + it('should return InvalidArgument putting object with ' + 'bad encoded tags', done => { + s3.send(new PutObjectCommand({ Bucket: bucket, Key: object, Tagging: 'key1==value1' })) + .then(() => { + assert(false, 'Expected failure but got success'); + }) + .catch(err => { + checkError(err, 'InvalidArgument', 400); + done(); + }); }); - it('should return InvalidArgument putting object tag with ' + - 'invalid characters: %', done => { + it('should return InvalidArgument putting object tag with ' + 'invalid characters: %', done => { const value = 'value1%'; - s3.send(new PutObjectCommand({ Bucket: bucket, Key: object, Tagging: - `key1=${value}` })).then(() => { - assert(false, 'Expected failure but got success'); - }).catch(err => { - checkError(err, 'InvalidArgument', 400); - done(); - }); + s3.send(new PutObjectCommand({ Bucket: bucket, Key: object, Tagging: `key1=${value}` })) + .then(() => { + assert(false, 'Expected failure but got success'); + }) + .catch(err => { + checkError(err, 'InvalidArgument', 400); + done(); + }); }); }); }); }); - describe('PUT object with object lock', () => { const bucket = 'bucket2putstuffin4324242-lock'; withV4(sigCfg => { @@ -323,54 +339,61 @@ describe('PUT object with object lock', () => { beforeEach(async () => { bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; - await s3.send(new CreateBucketCommand({ - Bucket: bucket, - ObjectLockEnabledForBucket: true, - })); + await s3.send( + new CreateBucketCommand({ + Bucket: bucket, + ObjectLockEnabledForBucket: true, + }), + ); }); afterEach(() => { process.stdout.write('Emptying bucket'); - return bucketUtil.empty(bucket) - .then(() => { - process.stdout.write('Deleting bucket'); - return bucketUtil.deleteOne(bucket); - }) - .catch(err => { - process.stdout.write('Error in afterEach'); - throw err; - }); + return bucketUtil + .empty(bucket) + .then(() => { + process.stdout.write('Deleting bucket'); + return bucketUtil.deleteOne(bucket); + }) + .catch(err => { + process.stdout.write('Error in afterEach'); + throw err; + }); }); - it('should put object with valid object lock retention date and ' + - 'mode when object lock is enabled on the bucket', done => { - const date = new Date(2050, 10, 10); - const params = { - Bucket: bucket, - Key: 'key1', - ObjectLockRetainUntilDate: date, - ObjectLockMode: 'COMPLIANCE', - }; - s3.send(new PutObjectCommand(params)).then(res => { - changeObjectLock( - [{ bucket, key: 'key1', versionId: res.VersionId }], '', done); - }); - }); + it( + 'should put object with valid object lock retention date and ' + + 'mode when object lock is enabled on the bucket', + done => { + const date = new Date(2050, 10, 10); + const params = { + Bucket: bucket, + Key: 'key1', + ObjectLockRetainUntilDate: date, + ObjectLockMode: 'COMPLIANCE', + }; + s3.send(new PutObjectCommand(params)).then(res => { + changeObjectLock([{ bucket, key: 'key1', versionId: res.VersionId }], '', done); + }); + }, + ); - it('should put object with valid object lock retention date and ' + - 'mode when object lock is enabled on the bucket', done => { - const date = new Date(2050, 10, 10); - const params = { - Bucket: bucket, - Key: 'key2', - ObjectLockRetainUntilDate: date, - ObjectLockMode: 'GOVERNANCE', - }; - s3.send(new PutObjectCommand(params)).then(res => { - changeObjectLock( - [{ bucket, key: 'key2', versionId: res.VersionId }], '', done); - }); - }); + it( + 'should put object with valid object lock retention date and ' + + 'mode when object lock is enabled on the bucket', + done => { + const date = new Date(2050, 10, 10); + const params = { + Bucket: bucket, + Key: 'key2', + ObjectLockRetainUntilDate: date, + ObjectLockMode: 'GOVERNANCE', + }; + s3.send(new PutObjectCommand(params)).then(res => { + changeObjectLock([{ bucket, key: 'key2', versionId: res.VersionId }], '', done); + }); + }, + ); it('should error with invalid object lock mode header', done => { const date = new Date(2050, 10, 10); @@ -380,13 +403,15 @@ describe('PUT object with object lock', () => { ObjectLockMode: 'Governance', ObjectLockRetainUntilDate: date, }; - s3.send(new PutObjectCommand(params)).then(() => { - assert(false, 'Expected failure but got success'); - }).catch(err => { - assert.strictEqual(err.name, 'InvalidArgument'); - assert(err.toString().includes('Unknown wormMode directive')); - done(); - }); + s3.send(new PutObjectCommand(params)) + .then(() => { + assert(false, 'Expected failure but got success'); + }) + .catch(err => { + assert.strictEqual(err.name, 'InvalidArgument'); + assert(err.toString().includes('Unknown wormMode directive')); + done(); + }); }); it('should put object with valid legal hold status ON', done => { @@ -396,8 +421,7 @@ describe('PUT object with object lock', () => { ObjectLockLegalHoldStatus: 'ON', }; s3.send(new PutObjectCommand(params)).then(res => { - changeObjectLock( - [{ bucket, key: 'key4', versionId: res.VersionId }], '', done); + changeObjectLock([{ bucket, key: 'key4', versionId: res.VersionId }], '', done); }); }); @@ -418,68 +442,83 @@ describe('PUT object with object lock', () => { Key: 'key6', ObjectLockLegalHoldStatus: 'on', }; - s3.send(new PutObjectCommand(params)).then(() => { - assert(false, 'Expected failure but got success'); - }).catch(err => { - assert.strictEqual(err.name, 'InvalidArgument'); - assert(err.toString().includes('Legal hold status must be one of "ON", "OFF"')); - done(); - }); + s3.send(new PutObjectCommand(params)) + .then(() => { + assert(false, 'Expected failure but got success'); + }) + .catch(err => { + assert.strictEqual(err.name, 'InvalidArgument'); + assert(err.toString().includes('Legal hold status must be one of "ON", "OFF"')); + done(); + }); }); - it('should return error when object lock retain until date header is ' + - 'provided but object lock mode header is missing', done => { - const date = new Date(2050, 10, 10); - const params = { - Bucket: bucket, - Key: 'key7', - ObjectLockRetainUntilDate: date, - }; - s3.send(new PutObjectCommand(params)).then(() => { - assert(false, 'Expected failure but got success'); - }).catch(err => { - const expectedErrMessage - = 'x-amz-object-lock-retain-until-date and ' + - 'x-amz-object-lock-mode must both be supplied'; - assert.strictEqual(err.name, 'InvalidArgument'); - assert(err.toString().includes(expectedErrMessage)); - done(); - }); - }); + it( + 'should return error when object lock retain until date header is ' + + 'provided but object lock mode header is missing', + done => { + const date = new Date(2050, 10, 10); + const params = { + Bucket: bucket, + Key: 'key7', + ObjectLockRetainUntilDate: date, + }; + s3.send(new PutObjectCommand(params)) + .then(() => { + assert(false, 'Expected failure but got success'); + }) + .catch(err => { + const expectedErrMessage = + 'x-amz-object-lock-retain-until-date and ' + 'x-amz-object-lock-mode must both be supplied'; + assert.strictEqual(err.name, 'InvalidArgument'); + assert(err.toString().includes(expectedErrMessage)); + done(); + }); + }, + ); - it('should return error when object lock mode header is provided ' + - 'but object lock retain until date header is missing', done => { - const params = { - Bucket: bucket, - Key: 'key8', - ObjectLockMode: 'GOVERNANCE', - }; - s3.send(new PutObjectCommand(params)).then(() => { - assert(false, 'Expected failure but got success'); - }).catch(err => { - const expectedErrMessage - = 'x-amz-object-lock-retain-until-date and ' + - 'x-amz-object-lock-mode must both be supplied'; - assert.strictEqual(err.name, 'InvalidArgument'); - assert(err.toString().includes(expectedErrMessage)); - done(); - }); - }); + it( + 'should return error when object lock mode header is provided ' + + 'but object lock retain until date header is missing', + done => { + const params = { + Bucket: bucket, + Key: 'key8', + ObjectLockMode: 'GOVERNANCE', + }; + s3.send(new PutObjectCommand(params)) + .then(() => { + assert(false, 'Expected failure but got success'); + }) + .catch(err => { + const expectedErrMessage = + 'x-amz-object-lock-retain-until-date and ' + 'x-amz-object-lock-mode must both be supplied'; + assert.strictEqual(err.name, 'InvalidArgument'); + assert(err.toString().includes(expectedErrMessage)); + done(); + }); + }, + ); - it('should return InvalidStorageClass error when x-amz-storage-class header is provided ' + - 'and not equal to STANDARD', done => { - const params = { - Bucket: bucket, - Key: 'key8', - StorageClass: 'COLD', - }; - s3.send(new PutObjectCommand(params)).then(() => { - assert(false, 'Expected failure but got success'); - }).catch(err => { - assert.strictEqual(err.name, 'InvalidStorageClass'); - assert.strictEqual(err.$metadata.httpStatusCode, 400); - done(); - }); - }); + it( + 'should return InvalidStorageClass error when x-amz-storage-class header is provided ' + + 'and not equal to STANDARD', + done => { + const params = { + Bucket: bucket, + Key: 'key8', + StorageClass: 'COLD', + }; + s3.send(new PutObjectCommand(params)) + .then(() => { + assert(false, 'Expected failure but got success'); + }) + .catch(err => { + assert.strictEqual(err.name, 'InvalidStorageClass'); + assert.strictEqual(err.$metadata.httpStatusCode, 400); + done(); + }); + }, + ); }); }); diff --git a/tests/functional/aws-node-sdk/test/object/putObjAcl.js b/tests/functional/aws-node-sdk/test/object/putObjAcl.js index aa8ce0d9a7..a44a6b02cd 100644 --- a/tests/functional/aws-node-sdk/test/object/putObjAcl.js +++ b/tests/functional/aws-node-sdk/test/object/putObjAcl.js @@ -1,15 +1,11 @@ const assert = require('assert'); -const { - PutObjectCommand, - PutObjectAclCommand, -} = require('@aws-sdk/client-s3'); +const { PutObjectCommand, PutObjectAclCommand } = require('@aws-sdk/client-s3'); const withV4 = require('../support/withV4'); const BucketUtility = require('../../lib/utility/bucket-util'); const constants = require('../../../../../constants'); -const notOwnerCanonicalID = '79a59df900b949e55d96a1e698fba' + - 'cedfd6e09d98eacf8f8d5218e7cd47ef2bf'; +const notOwnerCanonicalID = '79a59df900b949e55d96a1e698fba' + 'cedfd6e09d98eacf8f8d5218e7cd47ef2bf'; const itSkipIfAWS = process.env.AWS_ON_AIR ? it.skip : it; class _AccessControlPolicy { @@ -66,31 +62,32 @@ describe('PUT Object ACL', () => { it('should put object ACLs', async () => { const s3 = bucketUtil.s3; const Bucket = bucketName; - const objects = [ - { Bucket, Key }, - ]; + const objects = [{ Bucket, Key }]; for (const param of objects) { await s3.send(new PutObjectCommand(param)); } - const data = await s3.send(new PutObjectAclCommand({ - Bucket, - Key, - ACL: 'public-read' - })); + const data = await s3.send( + new PutObjectAclCommand({ + Bucket, + Key, + ACL: 'public-read', + }), + ); assert(data); - }); + }); - it('should return NoSuchKey if try to put object ACLs ' + - 'for nonexistent object', async () => { + it('should return NoSuchKey if try to put object ACLs ' + 'for nonexistent object', async () => { const s3 = bucketUtil.s3; const Bucket = bucketName; try { - await s3.send(new PutObjectAclCommand({ - Bucket, - Key, - ACL: 'public-read' - })); + await s3.send( + new PutObjectAclCommand({ + Bucket, + Key, + ACL: 'public-read', + }), + ); throw new Error('Expected NoSuchKey error'); } catch (err) { assert(err); @@ -103,25 +100,23 @@ describe('PUT Object ACL', () => { before(async () => { await s3.send(new PutObjectCommand({ Bucket: bucketName, Key })); }); - + after(async () => { process.stdout.write('deleting bucket'); await bucketUtil.empty(bucketName); }); - + // The supplied canonical ID is not associated with a real AWS // account, so AWS_ON_AIR will raise a 400 InvalidArgument - itSkipIfAWS('should return AccessDenied if try to change owner ' + - 'ID in ACL request body', async () => { - const acp = new _AccessControlPolicy( - { ownerID: notOwnerCanonicalID }); + itSkipIfAWS('should return AccessDenied if try to change owner ' + 'ID in ACL request body', async () => { + const acp = new _AccessControlPolicy({ ownerID: notOwnerCanonicalID }); acp.addGrantee('Group', constants.publicId, 'READ'); const putAclParams = { Bucket: bucketName, Key, AccessControlPolicy: acp, }; - + try { await s3.send(new PutObjectAclCommand(putAclParams)); throw new Error('Expected AccessDenied error'); diff --git a/tests/functional/aws-node-sdk/test/object/putObjTagging.js b/tests/functional/aws-node-sdk/test/object/putObjTagging.js index a86e089ca4..f4b6ac37ab 100644 --- a/tests/functional/aws-node-sdk/test/object/putObjTagging.js +++ b/tests/functional/aws-node-sdk/test/object/putObjTagging.js @@ -16,11 +16,14 @@ const bucketName = 'testputtaggingbucket'; const objectName = 'testputtaggingobject'; const objectNameAcl = 'testputtaggingobjectacl'; -const taggingConfig = { TagSet: [ - { - Key: 'key1', - Value: 'value1', - }] }; +const taggingConfig = { + TagSet: [ + { + Key: 'key1', + Value: 'value1', + }, + ], +}; function generateMultipleTagConfig(number) { const tags = []; @@ -61,28 +64,29 @@ describe('PUT object taggings', () => { taggingTests.forEach(taggingTest => { it(taggingTest.it, async () => { - const taggingConfig = generateTaggingConfig( - taggingTest.tag.key, - taggingTest.tag.value - ); - + const taggingConfig = generateTaggingConfig(taggingTest.tag.key, taggingTest.tag.value); + if (taggingTest.error) { try { - await s3.send(new PutObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - Tagging: taggingConfig - })); + await s3.send( + new PutObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + Tagging: taggingConfig, + }), + ); assert.fail('Expected an error but request succeeded'); } catch (err) { checkError(err, taggingTest.error, 400); } } else { - const data = await s3.send(new PutObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - Tagging: taggingConfig - })); + const data = await s3.send( + new PutObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + Tagging: taggingConfig, + }), + ); assert.strictEqual(Object.keys(data).length, 1); } }); @@ -90,59 +94,70 @@ describe('PUT object taggings', () => { it('should allow putting 50 tags', async () => { const taggingConfig = generateMultipleTagConfig(50); - await s3.send(new PutObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - Tagging: taggingConfig - })); + await s3.send( + new PutObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + Tagging: taggingConfig, + }), + ); }); it('should return BadRequest if putting more than 50 tags', async () => { const taggingConfig = generateMultipleTagConfig(51); try { - await s3.send(new PutObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - Tagging: taggingConfig - })); + await s3.send( + new PutObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + Tagging: taggingConfig, + }), + ); assert.fail('Expected BadRequest error'); } catch (err) { checkError(err, 'BadRequest', 400); } }); - it('should put tag set', async () => { - await s3.send(new PutObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - Tagging: taggingConfig, - })); + await s3.send( + new PutObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + Tagging: taggingConfig, + }), + ); - const data = await s3.send(new GetObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - })); + const data = await s3.send( + new GetObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); assert.deepStrictEqual(data.TagSet, taggingConfig.TagSet); }); it('should return InvalidTag if using the same key twice', async () => { try { - await s3.send(new PutObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - Tagging: { TagSet: [ - { - Key: 'key1', - Value: 'value1', - }, - { - Key: 'key1', - Value: 'value2', + await s3.send( + new PutObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + Tagging: { + TagSet: [ + { + Key: 'key1', + Value: 'value1', + }, + { + Key: 'key1', + Value: 'value2', + }, + ], }, - ] }, - })); + }), + ); throw new Error('Expected InvalidRequest error'); } catch (err) { checkError(err, 'InvalidTag', 400); @@ -151,18 +166,20 @@ describe('PUT object taggings', () => { it('should return InvalidTag if key is an empty string', async () => { try { - await s3.send(new PutObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - Tagging: { - TagSet: [ - { - Key: '', - Value: 'value1', - }, - ] - } - })); + await s3.send( + new PutObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + Tagging: { + TagSet: [ + { + Key: '', + Value: 'value1', + }, + ], + }, + }), + ); assert.fail('Expected InvalidTag error'); } catch (err) { checkError(err, 'InvalidTag', 400); @@ -170,103 +187,126 @@ describe('PUT object taggings', () => { }); it('should be able to put an empty Tag set', async () => { - const data = await s3.send(new PutObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - Tagging: { TagSet: [] } - })); + const data = await s3.send( + new PutObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + Tagging: { TagSet: [] }, + }), + ); assert.strictEqual(data.$metadata.httpStatusCode, 200); }); - it('should return NoSuchKey put tag to a non-existing object', - async () => { + it('should return NoSuchKey put tag to a non-existing object', async () => { try { - await s3.send(new PutObjectTaggingCommand({ - Bucket: bucketName, - Key: 'nonexisting', - Tagging: taggingConfig, - })); + await s3.send( + new PutObjectTaggingCommand({ + Bucket: bucketName, + Key: 'nonexisting', + Tagging: taggingConfig, + }), + ); throw new Error('Expected NoSuchKey error'); } catch (err) { checkError(err, 'NoSuchKey', 404); } }); - it('should return 403 AccessDenied putting tag with another account', - async () => { + it('should return 403 AccessDenied putting tag with another account', async () => { try { - await otherAccountS3.send(new PutObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - Tagging: taggingConfig, - })); + await otherAccountS3.send( + new PutObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + Tagging: taggingConfig, + }), + ); throw new Error('Expected AccessDenied error'); } catch (err) { checkError(err, 'AccessDenied', 403); } }); - it('should return 403 AccessDenied putting tag with a different ' + - 'account to an object with ACL "public-read-write"', - async () => { - await s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName, - ACL: 'public-read-write', - })); + it( + 'should return 403 AccessDenied putting tag with a different ' + + 'account to an object with ACL "public-read-write"', + async () => { + await s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + ACL: 'public-read-write', + }), + ); - try { - await otherAccountS3.send(new PutObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - Tagging: taggingConfig, - })); - throw new Error('Expected AccessDenied error'); - } catch (err) { - checkError(err, 'AccessDenied', 403); - } - }); + try { + await otherAccountS3.send( + new PutObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + Tagging: taggingConfig, + }), + ); + throw new Error('Expected AccessDenied error'); + } catch (err) { + checkError(err, 'AccessDenied', 403); + } + }, + ); - it('should return 403 AccessDenied putting tag to an object ' + - ' in a bucket created with a different account', - async () => { - await s3.send(new PutBucketAclCommand({ - Bucket: bucketName, - ACL: 'public-read-write', - })); - await otherAccountS3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectNameAcl, - })); + it( + 'should return 403 AccessDenied putting tag to an object ' + + ' in a bucket created with a different account', + async () => { + await s3.send( + new PutBucketAclCommand({ + Bucket: bucketName, + ACL: 'public-read-write', + }), + ); + await otherAccountS3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectNameAcl, + }), + ); - try { - await otherAccountS3.send(new PutObjectTaggingCommand({ + try { + await otherAccountS3.send( + new PutObjectTaggingCommand({ + Bucket: bucketName, + Key: objectNameAcl, + Tagging: taggingConfig, + }), + ); + throw new Error('Expected AccessDenied error'); + } catch (err) { + checkError(err, 'AccessDenied', 403); + } + }, + ); + + it('should put tag to an object in a bucket created with same ' + 'account', async () => { + await s3.send( + new PutBucketAclCommand({ + Bucket: bucketName, + ACL: 'public-read-write', + }), + ); + await otherAccountS3.send( + new PutObjectCommand({ Bucket: bucketName, Key: objectNameAcl, - Tagging: taggingConfig, - })); - throw new Error('Expected AccessDenied error'); - } catch (err) { - checkError(err, 'AccessDenied', 403); - } - }); - - it('should put tag to an object in a bucket created with same ' + - 'account', async () => { - await s3.send(new PutBucketAclCommand({ - Bucket: bucketName, - ACL: 'public-read-write', - })); - await otherAccountS3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectNameAcl, - })); + }), + ); - await s3.send(new PutObjectTaggingCommand({ - Bucket: bucketName, - Key: objectNameAcl, - Tagging: taggingConfig, - })); + await s3.send( + new PutObjectTaggingCommand({ + Bucket: bucketName, + Key: objectNameAcl, + Tagging: taggingConfig, + }), + ); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/object/putObjectLegalHold.js b/tests/functional/aws-node-sdk/test/object/putObjectLegalHold.js index 7dc9a2c23c..a263844a5f 100644 --- a/tests/functional/aws-node-sdk/test/object/putObjectLegalHold.js +++ b/tests/functional/aws-node-sdk/test/object/putObjectLegalHold.js @@ -48,7 +48,6 @@ function createLegalHoldParams(bucket, key, status, versionId) { return params; } - describe('PUT object legal hold', () => { withV4(sigCfg => { const bucketUtil = new BucketUtility('default', sigCfg); @@ -58,10 +57,12 @@ describe('PUT object legal hold', () => { let versionId; beforeEach(async () => { - await s3.send(new CreateBucketCommand({ - Bucket: bucket, - ObjectLockEnabledForBucket: true, - })); + await s3.send( + new CreateBucketCommand({ + Bucket: bucket, + ObjectLockEnabledForBucket: true, + }), + ); await s3.send(new CreateBucketCommand({ Bucket: unlockedBucket })); await s3.send(new PutObjectCommand({ Bucket: unlockedBucket, Key: key })); await s3.send(new PutObjectCommand({ Bucket: bucket, Key: key })); @@ -76,71 +77,83 @@ describe('PUT object legal hold', () => { resolve(); }); }) - .then(() => bucketUtil.empty(bucket, true)) - .then(() => bucketUtil.empty(unlockedBucket, true)) - .then(() => bucketUtil.deleteMany([bucket, unlockedBucket])); + .then(() => bucketUtil.empty(bucket, true)) + .then(() => bucketUtil.empty(unlockedBucket, true)) + .then(() => bucketUtil.deleteMany([bucket, unlockedBucket])); }); - it('should return AccessDenied putting legal hold with another account', - done => { + it('should return AccessDenied putting legal hold with another account', done => { const params = createLegalHoldParams(bucket, key, 'ON'); - otherAccountS3.send(new PutObjectLegalHoldCommand(params)).then(() => { - throw new Error('Expected AccessDenied error'); - }).catch(err => { - checkError(err, 'AccessDenied', 403); - done(); - }); + otherAccountS3 + .send(new PutObjectLegalHoldCommand(params)) + .then(() => { + throw new Error('Expected AccessDenied error'); + }) + .catch(err => { + checkError(err, 'AccessDenied', 403); + done(); + }); }); it('should return NoSuchKey error if key does not exist', done => { const params = createLegalHoldParams(bucket, 'keynotexist', 'ON'); - s3.send(new PutObjectLegalHoldCommand(params)).then(() => { - throw new Error('Expected NoSuchKey error'); - }).catch(err => { - checkError(err, 'NoSuchKey', 404); - done(); - }); + s3.send(new PutObjectLegalHoldCommand(params)) + .then(() => { + throw new Error('Expected NoSuchKey error'); + }) + .catch(err => { + checkError(err, 'NoSuchKey', 404); + done(); + }); }); it('should return NoSuchVersion error if version does not exist', done => { - s3.send(new PutObjectLegalHoldCommand({ - Bucket: bucket, - Key: key, - VersionId: '012345678901234567890123456789012', - LegalHold: mockLegalHold.on, - })).then(() => { - throw new Error('Expected NoSuchVersion error'); - }).catch(err => { - checkError(err, 'NoSuchVersion', 404); - done(); - }); + s3.send( + new PutObjectLegalHoldCommand({ + Bucket: bucket, + Key: key, + VersionId: '012345678901234567890123456789012', + LegalHold: mockLegalHold.on, + }), + ) + .then(() => { + throw new Error('Expected NoSuchVersion error'); + }) + .catch(err => { + checkError(err, 'NoSuchVersion', 404); + done(); + }); }); - it('should return InvalidRequest error putting legal hold to object ' + - 'in bucket with no object lock enabled', done => { - const params = createLegalHoldParams(unlockedBucket, key, 'ON'); - s3.send(new PutObjectLegalHoldCommand(params)).then(() => { - throw new Error('Expected InvalidRequest error'); - }).catch(err => { - checkError(err, 'InvalidRequest', 400); - done(); - }); - }); + it( + 'should return InvalidRequest error putting legal hold to object ' + + 'in bucket with no object lock enabled', + done => { + const params = createLegalHoldParams(unlockedBucket, key, 'ON'); + s3.send(new PutObjectLegalHoldCommand(params)) + .then(() => { + throw new Error('Expected InvalidRequest error'); + }) + .catch(err => { + checkError(err, 'InvalidRequest', 400); + done(); + }); + }, + ); - it('should return MethodNotAllowed if object version is delete marker', - done => { + it('should return MethodNotAllowed if object version is delete marker', done => { s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: key })) - .then(() => { - const params = createLegalHoldParams(bucket, key, 'ON'); - return s3.send(new PutObjectLegalHoldCommand(params)); - }) - .then(() => { - throw new Error('Expected MethodNotAllowed error'); - }) - .catch(err => { - checkError(err, 'MethodNotAllowed', 405); - done(); - }); + .then(() => { + const params = createLegalHoldParams(bucket, key, 'ON'); + return s3.send(new PutObjectLegalHoldCommand(params)); + }) + .then(() => { + throw new Error('Expected MethodNotAllowed error'); + }) + .catch(err => { + checkError(err, 'MethodNotAllowed', 405); + done(); + }); }); it('should put object legal hold ON', done => { @@ -150,55 +163,64 @@ describe('PUT object legal hold', () => { }); }); - it('should put object legal hold OFF', done => { const params = createLegalHoldParams(bucket, key, 'OFF'); s3.send(new PutObjectLegalHoldCommand(params)).then(() => { - changeObjectLock([{ bucket, key, versionId }], '', done); + changeObjectLock([{ bucket, key, versionId }], '', done); }); }); it('should return error if request has empty or undefined Status', done => { const params = createLegalHoldParams(bucket, key, ''); - s3.send(new PutObjectLegalHoldCommand(params)).then(() => { - throw new Error('Expected MalformedXML error'); - }).catch(err => { - checkError(err, 'MalformedXML', 400); - changeObjectLock([{ bucket, key, versionId }], '', done); - }); + s3.send(new PutObjectLegalHoldCommand(params)) + .then(() => { + throw new Error('Expected MalformedXML error'); + }) + .catch(err => { + checkError(err, 'MalformedXML', 400); + changeObjectLock([{ bucket, key, versionId }], '', done); + }); }); it('should return error if request does not contain Status', done => { - s3.send(new PutObjectLegalHoldCommand({ - Bucket: bucket, - Key: key, - LegalHold: {}, - })).then(() => { - throw new Error('Expected MalformedXML error'); - }).catch(err => { - checkError(err, 'MalformedXML', 400); - changeObjectLock([{ bucket, key, versionId }], '', done); - }); + s3.send( + new PutObjectLegalHoldCommand({ + Bucket: bucket, + Key: key, + LegalHold: {}, + }), + ) + .then(() => { + throw new Error('Expected MalformedXML error'); + }) + .catch(err => { + checkError(err, 'MalformedXML', 400); + changeObjectLock([{ bucket, key, versionId }], '', done); + }); }); it('expects params.LegalHold.Status to be a string', done => { const params = createLegalHoldParams(bucket, key, true); - s3.send(new PutObjectLegalHoldCommand(params)).then(() => { - throw new Error('Expected InvalidParameterType error'); - }).catch(err => { - checkError(err, 'MalformedXML', 400); - changeObjectLock([{ bucket, key, versionId }], '', done); - }); + s3.send(new PutObjectLegalHoldCommand(params)) + .then(() => { + throw new Error('Expected InvalidParameterType error'); + }) + .catch(err => { + checkError(err, 'MalformedXML', 400); + changeObjectLock([{ bucket, key, versionId }], '', done); + }); }); it('expects Status request xml must be one of "ON", "OFF"', done => { const params = createLegalHoldParams(bucket, key, 'on'); - s3.send(new PutObjectLegalHoldCommand(params)).then(() => { - throw new Error('Expected MalformedXML error'); - }).catch(err => { - checkError(err, 'MalformedXML', 400); - changeObjectLock([{ bucket, key, versionId }], '', done); - }); + s3.send(new PutObjectLegalHoldCommand(params)) + .then(() => { + throw new Error('Expected MalformedXML error'); + }) + .catch(err => { + checkError(err, 'MalformedXML', 400); + changeObjectLock([{ bucket, key, versionId }], '', done); + }); }); it('should support request with versionId parameter', done => { @@ -232,7 +254,8 @@ describe('PUT object legal hold iam action and version id', () => { const unauthBucketUtil = new BucketUtility('default', sigCfg, true); const unauthS3 = unauthBucketUtil.s3; const CommandClass = eval(operation); - unauthS3.send(new CommandClass(params)) + unauthS3 + .send(new CommandClass(params)) .then(data => callback(null, data)) .catch(err => callback(err)); } @@ -254,18 +277,21 @@ describe('PUT object legal hold iam action and version id', () => { beforeEach(() => { process.stdout.write('Setting up bucket policy legal hold tests\n'); - return s3.send(new CreateBucketCommand({ - Bucket: testBucket, - ObjectLockEnabledForBucket: true, - })) - .then(() => s3.send(new PutObjectCommand({ Bucket: testBucket, Key: key }))) - .then(res => { - versionId = res.VersionId; - }) - .catch(err => { - process.stdout.write('Error in beforeEach\n'); - throw err; - }); + return s3 + .send( + new CreateBucketCommand({ + Bucket: testBucket, + ObjectLockEnabledForBucket: true, + }), + ) + .then(() => s3.send(new PutObjectCommand({ Bucket: testBucket, Key: key }))) + .then(res => { + versionId = res.VersionId; + }) + .catch(err => { + process.stdout.write('Error in beforeEach\n'); + throw err; + }); }); afterEach(async () => { @@ -300,30 +326,41 @@ describe('PUT object legal hold iam action and version id', () => { Version: '2012-10-17', Statement: [statement], }; - s3.send(new PutBucketPolicyCommand({ - Bucket: testBucket, - Policy: JSON.stringify(bucketPolicy), - })).then(() => { - done(); - }).catch(err => { - assert.ifError(err); - done(); - }); + s3.send( + new PutBucketPolicyCommand({ + Bucket: testBucket, + Policy: JSON.stringify(bucketPolicy), + }), + ) + .then(() => { + done(); + }) + .catch(err => { + assert.ifError(err); + done(); + }); }); if (testCase.expectedResult === 'allow') { afterEach(() => - s3.send(new PutObjectLegalHoldCommand({ - Bucket: testBucket, - Key: key, - LegalHold: { Status: 'OFF' }, - })) - .then(() => s3.send(new PutObjectLegalHoldCommand({ - Bucket: testBucket, - Key: key, - VersionId: versionId, - LegalHold: { Status: 'OFF' }, - }))) + s3 + .send( + new PutObjectLegalHoldCommand({ + Bucket: testBucket, + Key: key, + LegalHold: { Status: 'OFF' }, + }), + ) + .then(() => + s3.send( + new PutObjectLegalHoldCommand({ + Bucket: testBucket, + Key: key, + VersionId: versionId, + LegalHold: { Status: 'OFF' }, + }), + ), + ), ); } diff --git a/tests/functional/aws-node-sdk/test/object/putPart.js b/tests/functional/aws-node-sdk/test/object/putPart.js index ec66956a03..17ca70e5d9 100644 --- a/tests/functional/aws-node-sdk/test/object/putPart.js +++ b/tests/functional/aws-node-sdk/test/object/putPart.js @@ -21,50 +21,56 @@ describe('PUT object', () => { beforeEach(async () => { bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; - + await s3.send(new CreateBucketCommand({ Bucket: bucket })); - const res = await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: key - })); + const res = await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + }), + ); uploadId = res.UploadId; }); afterEach(async () => { - await s3.send(new AbortMultipartUploadCommand({ - Bucket: bucket, - Key: key, - UploadId: uploadId, - })); + await s3.send( + new AbortMultipartUploadCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + }), + ); await bucketUtil.empty(bucket); await bucketUtil.deleteOne(bucket); }); - it('should return Not Implemented error for obj. encryption using ' + - 'customer-provided encryption keys', async () => { - const params = { - Bucket: bucket, - Key: 'key', - PartNumber: 0, - UploadId: uploadId, - SSECustomerAlgorithm: 'AES256' - }; - try { - await s3.send(new UploadPartCommand(params)); - throw new Error('Expected NotImplemented error'); - } catch (err) { - assert.strictEqual(err.name, 'NotImplemented'); - } - }); + it( + 'should return Not Implemented error for obj. encryption using ' + 'customer-provided encryption keys', + async () => { + const params = { + Bucket: bucket, + Key: 'key', + PartNumber: 0, + UploadId: uploadId, + SSECustomerAlgorithm: 'AES256', + }; + try { + await s3.send(new UploadPartCommand(params)); + throw new Error('Expected NotImplemented error'); + } catch (err) { + assert.strictEqual(err.name, 'NotImplemented'); + } + }, + ); it('should return InvalidArgument if negative PartNumber', async () => { const params = { Bucket: bucket, Key: 'key', PartNumber: -1, - UploadId: uploadId + UploadId: uploadId, }; - + try { await s3.send(new UploadPartCommand(params)); assert.fail('Expected InvalidArgument error'); diff --git a/tests/functional/aws-node-sdk/test/object/putRetention.js b/tests/functional/aws-node-sdk/test/object/putRetention.js index a01d163724..9a8e940945 100644 --- a/tests/functional/aws-node-sdk/test/object/putRetention.js +++ b/tests/functional/aws-node-sdk/test/object/putRetention.js @@ -6,7 +6,7 @@ const { PutObjectCommand, DeleteObjectCommand, PutObjectRetentionCommand, - PutBucketPolicyCommand + PutBucketPolicyCommand, } = require('@aws-sdk/client-s3'); const { errorInstances } = require('arsenal'); @@ -24,7 +24,6 @@ const retentionConfig = { RetainUntilDate: moment().add(1, 'd').add(123, 'ms').toDate(), }; - const changeObjectLockPromise = promisify(changeObjectLock); describe('PUT object retention', () => { @@ -36,10 +35,12 @@ describe('PUT object retention', () => { let versionId; beforeEach(async () => { - await s3.send(new CreateBucketCommand({ - Bucket: bucketName, - ObjectLockEnabledForBucket: true, - })); + await s3.send( + new CreateBucketCommand({ + Bucket: bucketName, + ObjectLockEnabledForBucket: true, + }), + ); await s3.send(new CreateBucketCommand({ Bucket: unlockedBucket })); await s3.send(new PutObjectCommand({ Bucket: unlockedBucket, Key: objectName })); const putRes = await s3.send(new PutObjectCommand({ Bucket: bucketName, Key: objectName })); @@ -54,11 +55,13 @@ describe('PUT object retention', () => { it('should return AccessDenied putting retention with another account', async () => { try { - await otherAccountS3.send(new PutObjectRetentionCommand({ - Bucket: bucketName, - Key: objectName, - Retention: retentionConfig, - })); + await otherAccountS3.send( + new PutObjectRetentionCommand({ + Bucket: bucketName, + Key: objectName, + Retention: retentionConfig, + }), + ); assert.fail('Expected error'); } catch (err) { checkError(err, 'AccessDenied', 403); @@ -67,11 +70,13 @@ describe('PUT object retention', () => { it('should return NoSuchKey error if key does not exist', async () => { try { - await s3.send(new PutObjectRetentionCommand({ - Bucket: bucketName, - Key: 'thiskeydoesnotexist', - Retention: retentionConfig, - })); + await s3.send( + new PutObjectRetentionCommand({ + Bucket: bucketName, + Key: 'thiskeydoesnotexist', + Retention: retentionConfig, + }), + ); assert.fail('Expected error'); } catch (err) { checkError(err, 'NoSuchKey', 404); @@ -80,40 +85,48 @@ describe('PUT object retention', () => { it('should return NoSuchVersion error if version does not exist', async () => { try { - await s3.send(new PutObjectRetentionCommand({ - Bucket: bucketName, - Key: objectName, - VersionId: '012345678901234567890123456789012', - Retention: retentionConfig, - })); + await s3.send( + new PutObjectRetentionCommand({ + Bucket: bucketName, + Key: objectName, + VersionId: '012345678901234567890123456789012', + Retention: retentionConfig, + }), + ); assert.fail('Expected error'); } catch (err) { checkError(err, 'NoSuchVersion', 404); } }); - it('should return InvalidRequest error putting retention to object in bucket with no object lock ' + - 'enabled', async () => { - try { - await s3.send(new PutObjectRetentionCommand({ - Bucket: unlockedBucket, - Key: objectName, - Retention: retentionConfig, - })); - assert.fail('Expected error'); - } catch (err) { - checkError(err, 'InvalidRequest', 400); - } - }); + it( + 'should return InvalidRequest error putting retention to object in bucket with no object lock ' + 'enabled', + async () => { + try { + await s3.send( + new PutObjectRetentionCommand({ + Bucket: unlockedBucket, + Key: objectName, + Retention: retentionConfig, + }), + ); + assert.fail('Expected error'); + } catch (err) { + checkError(err, 'InvalidRequest', 400); + } + }, + ); it('should return MethodNotAllowed if object version is delete marker', async () => { await s3.send(new DeleteObjectCommand({ Bucket: bucketName, Key: objectName })); try { - await s3.send(new PutObjectRetentionCommand({ - Bucket: bucketName, - Key: objectName, - Retention: retentionConfig, - })); + await s3.send( + new PutObjectRetentionCommand({ + Bucket: bucketName, + Key: objectName, + Retention: retentionConfig, + }), + ); assert.fail('Expected error'); } catch (err) { checkError(err, 'MethodNotAllowed', 405); @@ -121,21 +134,25 @@ describe('PUT object retention', () => { }); it('should put object retention', async () => { - await s3.send(new PutObjectRetentionCommand({ - Bucket: bucketName, - Key: objectName, - Retention: retentionConfig, - })); + await s3.send( + new PutObjectRetentionCommand({ + Bucket: bucketName, + Key: objectName, + Retention: retentionConfig, + }), + ); await changeObjectLockPromise([{ bucket: bucketName, key: objectName, versionId }], ''); }); it('should support request with versionId parameter', async () => { - await s3.send(new PutObjectRetentionCommand({ - Bucket: bucketName, - Key: objectName, - Retention: retentionConfig, - VersionId: versionId, - })); + await s3.send( + new PutObjectRetentionCommand({ + Bucket: bucketName, + Key: objectName, + Retention: retentionConfig, + VersionId: versionId, + }), + ); await changeObjectLockPromise([{ bucket: bucketName, key: objectName, versionId }], ''); }); }); @@ -161,13 +178,13 @@ describe('PUT object retention iam action and version id', () => { const unauthBucketUtil = new BucketUtility('default', sigCfg, true); const unauthS3 = unauthBucketUtil.s3; const CommandClass = eval(operation); - unauthS3.send(new CommandClass(params)) + unauthS3 + .send(new CommandClass(params)) .then(data => callback(null, data)) .catch(err => callback(err)); } } - function cbNoError(done) { return err => { assert.ifError(err); @@ -183,10 +200,12 @@ describe('PUT object retention iam action and version id', () => { } beforeEach(async () => { - await s3.send(new CreateBucketCommand({ - Bucket: testBucket, - ObjectLockEnabledForBucket: true, - })); + await s3.send( + new CreateBucketCommand({ + Bucket: testBucket, + ObjectLockEnabledForBucket: true, + }), + ); const res = await s3.send(new PutObjectCommand({ Bucket: testBucket, Key: objectName })); versionId = res.VersionId; }); @@ -223,15 +242,19 @@ describe('PUT object retention iam action and version id', () => { Version: '2012-10-17', Statement: [statement], }; - s3.send(new PutBucketPolicyCommand({ - Bucket: testBucket, - Policy: JSON.stringify(bucketPolicy), - })).then(() => { - done(); - }).catch(err => { - assert.ifError(err); - done(); - }); + s3.send( + new PutBucketPolicyCommand({ + Bucket: testBucket, + Policy: JSON.stringify(bucketPolicy), + }), + ) + .then(() => { + done(); + }) + .catch(err => { + assert.ifError(err); + done(); + }); }); it(`should ${testCase.expectedResult} unauthenticated putObjectRetention without VersionId`, done => { diff --git a/tests/functional/aws-node-sdk/test/object/putVersion.js b/tests/functional/aws-node-sdk/test/object/putVersion.js index 4b51c3608a..6af07afdf3 100644 --- a/tests/functional/aws-node-sdk/test/object/putVersion.js +++ b/tests/functional/aws-node-sdk/test/object/putVersion.js @@ -8,20 +8,20 @@ const { DummyRequestLogger } = require('../../../../unit/helpers'); const checkError = require('../../lib/utility/checkError'); const { getMetadata, fakeMetadataArchive, isNullKeyMetadataV1 } = require('../utils/init'); const { hasColdStorage } = require('../../lib/utility/test-utils'); -const { CreateBucketCommand, - PutObjectCommand, - HeadObjectCommand, +const { + CreateBucketCommand, + PutObjectCommand, + HeadObjectCommand, GetObjectCommand, PutObjectAclCommand, PutObjectTaggingCommand, PutObjectLegalHoldCommand, ListObjectsCommand, DeleteObjectCommand, - PutBucketVersioningCommand } = require('@aws-sdk/client-s3'); + PutBucketVersioningCommand, +} = require('@aws-sdk/client-s3'); -const { - LOCATION_NAME_DMF, -} = require('../../../../constants'); +const { LOCATION_NAME_DMF } = require('../../../../constants'); const log = new DummyRequestLogger(); const bucketName = 'bucket1putversion32'; @@ -46,7 +46,7 @@ function putObjectVersion(s3, params, vid, cb) { { step: 'build', name: 'addVersionIdHeader', // Add a name to identify the middleware - } + }, ); const promise = s3.send(command); @@ -67,7 +67,6 @@ function clearRestoreStatus(versions) { return versions; } - function checkVersionsAndUpdate(versionsBefore, versionsAfter, indexes) { indexes.forEach(i => { assert.notStrictEqual(versionsAfter[i].value.Size, versionsBefore[i].value.Size); @@ -98,16 +97,22 @@ describe('PUT object with x-scal-s3-version-id header', () => { beforeEach(done => { bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; - async.series([ - next => metadata.setup(next), - next => s3.send(new CreateBucketCommand({ Bucket: bucketName })).then(() => { - next(); - }), - next => s3.send(new CreateBucketCommand({ Bucket: bucketNameMD, - ObjectLockEnabledForBucket: true })).then(() => { - next(); - }), - ], done); + async.series( + [ + next => metadata.setup(next), + next => + s3.send(new CreateBucketCommand({ Bucket: bucketName })).then(() => { + next(); + }), + next => + s3 + .send(new CreateBucketCommand({ Bucket: bucketNameMD, ObjectLockEnabledForBucket: true })) + .then(() => { + next(); + }), + ], + done, + ); }); afterEach(async () => { @@ -121,35 +126,43 @@ describe('PUT object with x-scal-s3-version-id header', () => { Bucket: bucketName, VersioningConfiguration: { Status: 'Enabled', - } + }, }; const params = { Bucket: bucketName, Key: objectName }; - async.series([ - next => s3.send(new PutBucketVersioningCommand(vParams)).then(() => next()), - next => s3.send(new PutObjectCommand(params)).then(() => next()), - next => putObjectVersion(s3, params, 'aJLWKz4Ko9IjBBgXKj5KQT.G9UHv0g7P', err => { - assert.strictEqual(err.name, 'InvalidArgument'); - assert.strictEqual(err.$metadata.httpStatusCode, 400); - return next(); - }), - ], err => { - assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); - return done(); - }); + async.series( + [ + next => s3.send(new PutBucketVersioningCommand(vParams)).then(() => next()), + next => s3.send(new PutObjectCommand(params)).then(() => next()), + next => + putObjectVersion(s3, params, 'aJLWKz4Ko9IjBBgXKj5KQT.G9UHv0g7P', err => { + assert.strictEqual(err.name, 'InvalidArgument'); + assert.strictEqual(err.$metadata.httpStatusCode, 400); + return next(); + }), + ], + err => { + assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); + return done(); + }, + ); }); it('should fail if key does not exist', done => { const params = { Bucket: bucketName, Key: objectName }; - async.series([ - next => putObjectVersion(s3, params, '', err => { - checkError(err, 'NoSuchKey', 404); - return next(); - }), - ], err => { - assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); - return done(); - }); + async.series( + [ + next => + putObjectVersion(s3, params, '', err => { + checkError(err, 'NoSuchKey', 404); + return next(); + }), + ], + err => { + assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); + return done(); + }, + ); }); it('should fail if version does not exist', done => { @@ -157,37 +170,49 @@ describe('PUT object with x-scal-s3-version-id header', () => { Bucket: bucketName, VersioningConfiguration: { Status: 'Enabled', - } + }, }; const params = { Bucket: bucketName, Key: objectName }; - async.series([ - next => s3.send(new PutBucketVersioningCommand(vParams)).then(() => next()), - next => s3.send(new PutObjectCommand(params)).then(() => next()), - next => putObjectVersion(s3, params, - '393833343735313131383832343239393939393952473030312020313031', err => { - checkError(err, 'NoSuchVersion', 404); - return next(); - }), - ], err => { - assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); - return done(); - }); + async.series( + [ + next => s3.send(new PutBucketVersioningCommand(vParams)).then(() => next()), + next => s3.send(new PutObjectCommand(params)).then(() => next()), + next => + putObjectVersion( + s3, + params, + '393833343735313131383832343239393939393952473030312020313031', + err => { + checkError(err, 'NoSuchVersion', 404); + return next(); + }, + ), + ], + err => { + assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); + return done(); + }, + ); }); it('should fail if archiving is not in progress', done => { const params = { Bucket: bucketName, Key: objectName }; - async.series([ - next => s3.send(new PutObjectCommand(params)).then(() => next()), - next => putObjectVersion(s3, params, '', err => { - checkError(err, 'InvalidObjectState', 403); - return next(); - }), - ], err => { - assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); - return done(); - }); + async.series( + [ + next => s3.send(new PutObjectCommand(params)).then(() => next()), + next => + putObjectVersion(s3, params, '', err => { + checkError(err, 'InvalidObjectState', 403); + return next(); + }), + ], + err => { + assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); + return done(); + }, + ); }); it('should fail if trying to overwrite a delete marker', done => { @@ -196,68 +221,88 @@ describe('PUT object with x-scal-s3-version-id header', () => { Bucket: bucketName, VersioningConfiguration: { Status: 'Enabled', - } + }, }; let vId; - async.series([ - next => s3.send(new PutBucketVersioningCommand(vParams)).then(() => next()), - next => s3.send(new PutObjectCommand(params)).then(() => next()), - next => s3.send(new DeleteObjectCommand(params)).then(res => { - vId = res.VersionId; - return next(); - }), - next => putObjectVersion(s3, params, vId, err => { - checkError(err, 'MethodNotAllowed', 405); - return next(); - }), - ], err => { - assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); - return done(); - }); + async.series( + [ + next => s3.send(new PutBucketVersioningCommand(vParams)).then(() => next()), + next => s3.send(new PutObjectCommand(params)).then(() => next()), + next => + s3.send(new DeleteObjectCommand(params)).then(res => { + vId = res.VersionId; + return next(); + }), + next => + putObjectVersion(s3, params, vId, err => { + checkError(err, 'MethodNotAllowed', 405); + return next(); + }), + ], + err => { + assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); + return done(); + }, + ); }); }); describeSkipNullMdV1('with cold storage location', () => { it('should overwrite an object', done => { - const params = { Bucket: bucketName, Key: objectName }; + const params = { Bucket: bucketName, Key: objectName }; let objMDBefore; let objMDAfter; let versionsBefore; let versionsAfter; - async.series([ - next => s3.send(new PutObjectCommand(params)).then(() => { - next(); - }), - next => fakeMetadataArchive(bucketName, objectName, undefined, archive, next), - next => getMetadata(bucketName, objectName, undefined, (err, objMD) => { - objMDBefore = objMD; - return next(err); - }), - next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => { - versionsBefore = clearRestoreStatus(res.Versions); - return next(err); - }), - next => putObjectVersion(s3, params, '', next), - next => getMetadata(bucketName, objectName, undefined, (err, objMD) => { - objMDAfter = objMD; - return next(err); - }), - next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => { - versionsAfter = clearRestoreStatus(res.Versions); - return next(err); - }), - ], err => { - assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); - - checkVersionsAndUpdate(versionsBefore, versionsAfter, [0]); - assert.deepStrictEqual(versionsAfter, versionsBefore); - - checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', - 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName', 'originOp']); - assert.deepStrictEqual(objMDAfter, objMDBefore); - return done(); - }); + async.series( + [ + next => + s3.send(new PutObjectCommand(params)).then(() => { + next(); + }), + next => fakeMetadataArchive(bucketName, objectName, undefined, archive, next), + next => + getMetadata(bucketName, objectName, undefined, (err, objMD) => { + objMDBefore = objMD; + return next(err); + }), + next => + metadata.listObject(bucketName, mdListingParams, log, (err, res) => { + versionsBefore = clearRestoreStatus(res.Versions); + return next(err); + }), + next => putObjectVersion(s3, params, '', next), + next => + getMetadata(bucketName, objectName, undefined, (err, objMD) => { + objMDAfter = objMD; + return next(err); + }), + next => + metadata.listObject(bucketName, mdListingParams, log, (err, res) => { + versionsAfter = clearRestoreStatus(res.Versions); + return next(err); + }), + ], + err => { + assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); + + checkVersionsAndUpdate(versionsBefore, versionsAfter, [0]); + assert.deepStrictEqual(versionsAfter, versionsBefore); + + checkObjMdAndUpdate(objMDBefore, objMDAfter, [ + 'location', + 'content-length', + 'microVersionId', + 'x-amz-restore', + 'archive', + 'dataStoreName', + 'originOp', + ]); + assert.deepStrictEqual(objMDAfter, objMDBefore); + return done(); + }, + ); }); it('should overwrite a version', done => { @@ -265,7 +310,7 @@ describe('PUT object with x-scal-s3-version-id header', () => { Bucket: bucketName, VersioningConfiguration: { Status: 'Enabled', - } + }, }; const params = { Bucket: bucketName, Key: objectName }; let objMDBefore; @@ -274,41 +319,56 @@ describe('PUT object with x-scal-s3-version-id header', () => { let versionsAfter; let vId; - async.series([ - next => s3.send(new PutBucketVersioningCommand(vParams)).then(() => next()), - next => s3.send(new PutObjectCommand(params)).then(res => { - vId = res.VersionId; - return next(); - }), - next => fakeMetadataArchive(bucketName, objectName, vId, archive, next), - next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => { - versionsBefore = clearRestoreStatus(res.Versions); - return next(err); - }), - next => getMetadata(bucketName, objectName, vId, (err, objMD) => { - objMDBefore = objMD; - return next(err); - }), - next => putObjectVersion(s3, params, vId, next), - next => getMetadata(bucketName, objectName, vId, (err, objMD) => { - objMDAfter = objMD; - return next(err); - }), - next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => { - versionsAfter = clearRestoreStatus(res.Versions); - return next(err); - }), - ], err => { - assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); - - checkVersionsAndUpdate(versionsBefore, versionsAfter, [0]); - assert.deepStrictEqual(versionsAfter, versionsBefore); - - checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'originOp', - 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName']); - assert.deepStrictEqual(objMDAfter, objMDBefore); - return done(); - }); + async.series( + [ + next => s3.send(new PutBucketVersioningCommand(vParams)).then(() => next()), + next => + s3.send(new PutObjectCommand(params)).then(res => { + vId = res.VersionId; + return next(); + }), + next => fakeMetadataArchive(bucketName, objectName, vId, archive, next), + next => + metadata.listObject(bucketName, mdListingParams, log, (err, res) => { + versionsBefore = clearRestoreStatus(res.Versions); + return next(err); + }), + next => + getMetadata(bucketName, objectName, vId, (err, objMD) => { + objMDBefore = objMD; + return next(err); + }), + next => putObjectVersion(s3, params, vId, next), + next => + getMetadata(bucketName, objectName, vId, (err, objMD) => { + objMDAfter = objMD; + return next(err); + }), + next => + metadata.listObject(bucketName, mdListingParams, log, (err, res) => { + versionsAfter = clearRestoreStatus(res.Versions); + return next(err); + }), + ], + err => { + assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); + + checkVersionsAndUpdate(versionsBefore, versionsAfter, [0]); + assert.deepStrictEqual(versionsAfter, versionsBefore); + + checkObjMdAndUpdate(objMDBefore, objMDAfter, [ + 'location', + 'content-length', + 'originOp', + 'microVersionId', + 'x-amz-restore', + 'archive', + 'dataStoreName', + ]); + assert.deepStrictEqual(objMDAfter, objMDBefore); + return done(); + }, + ); }); it('should overwrite the current version if empty version id header', done => { @@ -316,7 +376,7 @@ describe('PUT object with x-scal-s3-version-id header', () => { Bucket: bucketName, VersioningConfiguration: { Status: 'Enabled', - } + }, }; const params = { Bucket: bucketName, Key: objectName }; let objMDBefore; @@ -325,41 +385,56 @@ describe('PUT object with x-scal-s3-version-id header', () => { let versionsAfter; let vId; - async.series([ - next => s3.send(new PutBucketVersioningCommand(vParams)).then(() => next()), - next => s3.send(new PutObjectCommand(params)).then(res => { - vId = res.VersionId; - return next(); - }), - next => fakeMetadataArchive(bucketName, objectName, vId, archive, next), - next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => { - versionsBefore = clearRestoreStatus(res.Versions); - return next(err); - }), - next => getMetadata(bucketName, objectName, vId, (err, objMD) => { - objMDBefore = objMD; - return next(err); - }), - next => putObjectVersion(s3, params, '', next), - next => getMetadata(bucketName, objectName, vId, (err, objMD) => { - objMDAfter = objMD; - return next(err); - }), - next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => { - versionsAfter = clearRestoreStatus(res.Versions); - return next(err); - }), - ], err => { - assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); - - checkVersionsAndUpdate(versionsBefore, versionsAfter, [0]); - assert.deepStrictEqual(versionsAfter, versionsBefore); - - checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'originOp', - 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName']); - assert.deepStrictEqual(objMDAfter, objMDBefore); - return done(); - }); + async.series( + [ + next => s3.send(new PutBucketVersioningCommand(vParams)).then(() => next()), + next => + s3.send(new PutObjectCommand(params)).then(res => { + vId = res.VersionId; + return next(); + }), + next => fakeMetadataArchive(bucketName, objectName, vId, archive, next), + next => + metadata.listObject(bucketName, mdListingParams, log, (err, res) => { + versionsBefore = clearRestoreStatus(res.Versions); + return next(err); + }), + next => + getMetadata(bucketName, objectName, vId, (err, objMD) => { + objMDBefore = objMD; + return next(err); + }), + next => putObjectVersion(s3, params, '', next), + next => + getMetadata(bucketName, objectName, vId, (err, objMD) => { + objMDAfter = objMD; + return next(err); + }), + next => + metadata.listObject(bucketName, mdListingParams, log, (err, res) => { + versionsAfter = clearRestoreStatus(res.Versions); + return next(err); + }), + ], + err => { + assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); + + checkVersionsAndUpdate(versionsBefore, versionsAfter, [0]); + assert.deepStrictEqual(versionsAfter, versionsBefore); + + checkObjMdAndUpdate(objMDBefore, objMDAfter, [ + 'location', + 'content-length', + 'originOp', + 'microVersionId', + 'x-amz-restore', + 'archive', + 'dataStoreName', + ]); + assert.deepStrictEqual(objMDAfter, objMDBefore); + return done(); + }, + ); }); it('should overwrite a non-current null version', done => { @@ -367,7 +442,7 @@ describe('PUT object with x-scal-s3-version-id header', () => { Bucket: bucketName, VersioningConfiguration: { Status: 'Enabled', - } + }, }; const params = { Bucket: bucketName, Key: objectName }; let versionsBefore; @@ -375,39 +450,53 @@ describe('PUT object with x-scal-s3-version-id header', () => { let objMDBefore; let objMDAfter; - async.series([ - next => s3.send(new PutObjectCommand(params)).then(() => next()), - next => s3.send(new PutBucketVersioningCommand(vParams)).then(() => next()), - next => s3.send(new PutObjectCommand(params)).then(() => next()), - next => fakeMetadataArchive(bucketName, objectName, 'null', archive, next), - next => getMetadata(bucketName, objectName, 'null', (err, objMD) => { - objMDBefore = objMD; - return next(err); - }), - next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => { - versionsBefore = clearRestoreStatus(res.Versions); - next(err); - }), - next => putObjectVersion(s3, params, 'null', next), - next => getMetadata(bucketName, objectName, 'null', (err, objMD) => { - objMDAfter = objMD; - return next(err); - }), - next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => { - versionsAfter = clearRestoreStatus(res.Versions); - return next(err); - }), - ], err => { - assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); - - checkVersionsAndUpdate(versionsBefore, versionsAfter, [1]); - assert.deepStrictEqual(versionsAfter, versionsBefore); - - checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'originOp', - 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName']); - assert.deepStrictEqual(objMDAfter, objMDBefore); - return done(); - }); + async.series( + [ + next => s3.send(new PutObjectCommand(params)).then(() => next()), + next => s3.send(new PutBucketVersioningCommand(vParams)).then(() => next()), + next => s3.send(new PutObjectCommand(params)).then(() => next()), + next => fakeMetadataArchive(bucketName, objectName, 'null', archive, next), + next => + getMetadata(bucketName, objectName, 'null', (err, objMD) => { + objMDBefore = objMD; + return next(err); + }), + next => + metadata.listObject(bucketName, mdListingParams, log, (err, res) => { + versionsBefore = clearRestoreStatus(res.Versions); + next(err); + }), + next => putObjectVersion(s3, params, 'null', next), + next => + getMetadata(bucketName, objectName, 'null', (err, objMD) => { + objMDAfter = objMD; + return next(err); + }), + next => + metadata.listObject(bucketName, mdListingParams, log, (err, res) => { + versionsAfter = clearRestoreStatus(res.Versions); + return next(err); + }), + ], + err => { + assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); + + checkVersionsAndUpdate(versionsBefore, versionsAfter, [1]); + assert.deepStrictEqual(versionsAfter, versionsBefore); + + checkObjMdAndUpdate(objMDBefore, objMDAfter, [ + 'location', + 'content-length', + 'originOp', + 'microVersionId', + 'x-amz-restore', + 'archive', + 'dataStoreName', + ]); + assert.deepStrictEqual(objMDAfter, objMDBefore); + return done(); + }, + ); }); it('should overwrite the lastest version and keep nullVersionId', done => { @@ -415,7 +504,7 @@ describe('PUT object with x-scal-s3-version-id header', () => { Bucket: bucketName, VersioningConfiguration: { Status: 'Enabled', - } + }, }; const params = { Bucket: bucketName, Key: objectName }; let versionsBefore; @@ -424,42 +513,57 @@ describe('PUT object with x-scal-s3-version-id header', () => { let objMDAfter; let vId; - async.series([ - next => s3.send(new PutObjectCommand(params)).then(() => next()), - next => s3.send(new PutBucketVersioningCommand(vParams)).then(() => next()), - next => s3.send(new PutObjectCommand(params)).then(res => { - vId = res.VersionId; - return next(); - }), - next => fakeMetadataArchive(bucketName, objectName, vId, archive, next), - next => getMetadata(bucketName, objectName, vId, (err, objMD) => { - objMDBefore = objMD; - return next(err); - }), - next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => { - versionsBefore = clearRestoreStatus(res.Versions); - return next(err); - }), - next => putObjectVersion(s3, params, vId, next), - next => getMetadata(bucketName, objectName, vId, (err, objMD) => { - objMDAfter = objMD; - return next(err); - }), - next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => { - versionsAfter = clearRestoreStatus(res.Versions); - return next(err); - }), - ], err => { - assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); - - checkVersionsAndUpdate(versionsBefore, versionsAfter, [0]); - assert.deepStrictEqual(versionsAfter, versionsBefore); - - checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'originOp', - 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName']); - assert.deepStrictEqual(objMDAfter, objMDBefore); - return done(); - }); + async.series( + [ + next => s3.send(new PutObjectCommand(params)).then(() => next()), + next => s3.send(new PutBucketVersioningCommand(vParams)).then(() => next()), + next => + s3.send(new PutObjectCommand(params)).then(res => { + vId = res.VersionId; + return next(); + }), + next => fakeMetadataArchive(bucketName, objectName, vId, archive, next), + next => + getMetadata(bucketName, objectName, vId, (err, objMD) => { + objMDBefore = objMD; + return next(err); + }), + next => + metadata.listObject(bucketName, mdListingParams, log, (err, res) => { + versionsBefore = clearRestoreStatus(res.Versions); + return next(err); + }), + next => putObjectVersion(s3, params, vId, next), + next => + getMetadata(bucketName, objectName, vId, (err, objMD) => { + objMDAfter = objMD; + return next(err); + }), + next => + metadata.listObject(bucketName, mdListingParams, log, (err, res) => { + versionsAfter = clearRestoreStatus(res.Versions); + return next(err); + }), + ], + err => { + assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); + + checkVersionsAndUpdate(versionsBefore, versionsAfter, [0]); + assert.deepStrictEqual(versionsAfter, versionsBefore); + + checkObjMdAndUpdate(objMDBefore, objMDAfter, [ + 'location', + 'content-length', + 'originOp', + 'microVersionId', + 'x-amz-restore', + 'archive', + 'dataStoreName', + ]); + assert.deepStrictEqual(objMDAfter, objMDBefore); + return done(); + }, + ); }); it('should overwrite a current null version', done => { @@ -467,13 +571,13 @@ describe('PUT object with x-scal-s3-version-id header', () => { Bucket: bucketName, VersioningConfiguration: { Status: 'Enabled', - } + }, }; const sParams = { Bucket: bucketName, VersioningConfiguration: { Status: 'Suspended', - } + }, }; const params = { Bucket: bucketName, Key: objectName }; let objMDBefore; @@ -481,40 +585,54 @@ describe('PUT object with x-scal-s3-version-id header', () => { let versionsBefore; let versionsAfter; - async.series([ - next => s3.send(new PutBucketVersioningCommand(vParams)).then(() => next()), - next => s3.send(new PutObjectCommand(params)).then(() => next()), - next => s3.send(new PutBucketVersioningCommand(sParams)).then(() => next()), - next => s3.send(new PutObjectCommand(params)).then(() => next()), - next => fakeMetadataArchive(bucketName, objectName, undefined, archive, next), - next => getMetadata(bucketName, objectName, undefined, (err, objMD) => { - objMDBefore = objMD; - return next(err); - }), - next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => { - versionsBefore = clearRestoreStatus(res.Versions); - return next(err); - }), - next => putObjectVersion(s3, params, '', next), - next => getMetadata(bucketName, objectName, undefined, (err, objMD) => { - objMDAfter = objMD; - return next(err); - }), - next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => { - versionsAfter = clearRestoreStatus(res.Versions); - return next(err); - }), - ], err => { - assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); - - checkVersionsAndUpdate(versionsBefore, versionsAfter, [0]); - assert.deepStrictEqual(versionsAfter, versionsBefore); - - checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'originOp', - 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName']); - assert.deepStrictEqual(objMDAfter, objMDBefore); - return done(); - }); + async.series( + [ + next => s3.send(new PutBucketVersioningCommand(vParams)).then(() => next()), + next => s3.send(new PutObjectCommand(params)).then(() => next()), + next => s3.send(new PutBucketVersioningCommand(sParams)).then(() => next()), + next => s3.send(new PutObjectCommand(params)).then(() => next()), + next => fakeMetadataArchive(bucketName, objectName, undefined, archive, next), + next => + getMetadata(bucketName, objectName, undefined, (err, objMD) => { + objMDBefore = objMD; + return next(err); + }), + next => + metadata.listObject(bucketName, mdListingParams, log, (err, res) => { + versionsBefore = clearRestoreStatus(res.Versions); + return next(err); + }), + next => putObjectVersion(s3, params, '', next), + next => + getMetadata(bucketName, objectName, undefined, (err, objMD) => { + objMDAfter = objMD; + return next(err); + }), + next => + metadata.listObject(bucketName, mdListingParams, log, (err, res) => { + versionsAfter = clearRestoreStatus(res.Versions); + return next(err); + }), + ], + err => { + assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); + + checkVersionsAndUpdate(versionsBefore, versionsAfter, [0]); + assert.deepStrictEqual(versionsAfter, versionsBefore); + + checkObjMdAndUpdate(objMDBefore, objMDAfter, [ + 'location', + 'content-length', + 'originOp', + 'microVersionId', + 'x-amz-restore', + 'archive', + 'dataStoreName', + ]); + assert.deepStrictEqual(objMDAfter, objMDBefore); + return done(); + }, + ); }); it('should overwrite a non-current version', done => { @@ -522,7 +640,7 @@ describe('PUT object with x-scal-s3-version-id header', () => { Bucket: bucketName, VersioningConfiguration: { Status: 'Enabled', - } + }, }; const params = { Bucket: bucketName, Key: objectName }; let objMDBefore; @@ -531,43 +649,58 @@ describe('PUT object with x-scal-s3-version-id header', () => { let versionsAfter; let vId; - async.series([ - next => s3.send(new PutBucketVersioningCommand(vParams)).then(() => next()), - next => s3.send(new PutObjectCommand(params)).then(() => next()), - next => s3.send(new PutObjectCommand(params)).then(res => { - vId = res.VersionId; - return next(); - }), - next => fakeMetadataArchive(bucketName, objectName, vId, archive, next), - next => s3.send(new PutObjectCommand(params)).then(() => next()), - next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => { - versionsBefore = clearRestoreStatus(res.Versions); - return next(err); - }), - next => getMetadata(bucketName, objectName, vId, (err, objMD) => { - objMDBefore = objMD; - return next(err); - }), - next => putObjectVersion(s3, params, vId, next), - next => getMetadata(bucketName, objectName, vId, (err, objMD) => { - objMDAfter = objMD; - return next(err); - }), - next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => { - versionsAfter = clearRestoreStatus(res.Versions); - return next(err); - }), - ], err => { - assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); - - checkVersionsAndUpdate(versionsBefore, versionsAfter, [1]); - assert.deepStrictEqual(versionsAfter, versionsBefore); - - checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'originOp', - 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName']); - assert.deepStrictEqual(objMDAfter, objMDBefore); - return done(); - }); + async.series( + [ + next => s3.send(new PutBucketVersioningCommand(vParams)).then(() => next()), + next => s3.send(new PutObjectCommand(params)).then(() => next()), + next => + s3.send(new PutObjectCommand(params)).then(res => { + vId = res.VersionId; + return next(); + }), + next => fakeMetadataArchive(bucketName, objectName, vId, archive, next), + next => s3.send(new PutObjectCommand(params)).then(() => next()), + next => + metadata.listObject(bucketName, mdListingParams, log, (err, res) => { + versionsBefore = clearRestoreStatus(res.Versions); + return next(err); + }), + next => + getMetadata(bucketName, objectName, vId, (err, objMD) => { + objMDBefore = objMD; + return next(err); + }), + next => putObjectVersion(s3, params, vId, next), + next => + getMetadata(bucketName, objectName, vId, (err, objMD) => { + objMDAfter = objMD; + return next(err); + }), + next => + metadata.listObject(bucketName, mdListingParams, log, (err, res) => { + versionsAfter = clearRestoreStatus(res.Versions); + return next(err); + }), + ], + err => { + assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); + + checkVersionsAndUpdate(versionsBefore, versionsAfter, [1]); + assert.deepStrictEqual(versionsAfter, versionsBefore); + + checkObjMdAndUpdate(objMDBefore, objMDAfter, [ + 'location', + 'content-length', + 'originOp', + 'microVersionId', + 'x-amz-restore', + 'archive', + 'dataStoreName', + ]); + assert.deepStrictEqual(objMDAfter, objMDBefore); + return done(); + }, + ); }); it('should overwrite the current version', done => { @@ -575,7 +708,7 @@ describe('PUT object with x-scal-s3-version-id header', () => { Bucket: bucketName, VersioningConfiguration: { Status: 'Enabled', - } + }, }; const params = { Bucket: bucketName, Key: objectName }; let objMDBefore; @@ -584,42 +717,57 @@ describe('PUT object with x-scal-s3-version-id header', () => { let versionsAfter; let vId; - async.series([ - next => s3.send(new PutBucketVersioningCommand(vParams)).then(() => next()), - next => s3.send(new PutObjectCommand(params)).then(() => next()), - next => s3.send(new PutObjectCommand(params)).then(res => { - vId = res.VersionId; - return next(); - }), - next => fakeMetadataArchive(bucketName, objectName, vId, archive, next), - next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => { - versionsBefore = clearRestoreStatus(res.Versions); - return next(err); - }), - next => getMetadata(bucketName, objectName, vId, (err, objMD) => { - objMDBefore = objMD; - return next(err); - }), - next => putObjectVersion(s3, params, vId, next), - next => getMetadata(bucketName, objectName, vId, (err, objMD) => { - objMDAfter = objMD; - return next(err); - }), - next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => { - versionsAfter = clearRestoreStatus(res.Versions); - return next(err); - }), - ], err => { - assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); - - checkVersionsAndUpdate(versionsBefore, versionsAfter, [0]); - assert.deepStrictEqual(versionsAfter, versionsBefore); - - checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'originOp', - 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName']); - assert.deepStrictEqual(objMDAfter, objMDBefore); - return done(); - }); + async.series( + [ + next => s3.send(new PutBucketVersioningCommand(vParams)).then(() => next()), + next => s3.send(new PutObjectCommand(params)).then(() => next()), + next => + s3.send(new PutObjectCommand(params)).then(res => { + vId = res.VersionId; + return next(); + }), + next => fakeMetadataArchive(bucketName, objectName, vId, archive, next), + next => + metadata.listObject(bucketName, mdListingParams, log, (err, res) => { + versionsBefore = clearRestoreStatus(res.Versions); + return next(err); + }), + next => + getMetadata(bucketName, objectName, vId, (err, objMD) => { + objMDBefore = objMD; + return next(err); + }), + next => putObjectVersion(s3, params, vId, next), + next => + getMetadata(bucketName, objectName, vId, (err, objMD) => { + objMDAfter = objMD; + return next(err); + }), + next => + metadata.listObject(bucketName, mdListingParams, log, (err, res) => { + versionsAfter = clearRestoreStatus(res.Versions); + return next(err); + }), + ], + err => { + assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); + + checkVersionsAndUpdate(versionsBefore, versionsAfter, [0]); + assert.deepStrictEqual(versionsAfter, versionsBefore); + + checkObjMdAndUpdate(objMDBefore, objMDAfter, [ + 'location', + 'content-length', + 'originOp', + 'microVersionId', + 'x-amz-restore', + 'archive', + 'dataStoreName', + ]); + assert.deepStrictEqual(objMDAfter, objMDBefore); + return done(); + }, + ); }); it('should overwrite the current version after bucket version suspended', done => { @@ -627,13 +775,13 @@ describe('PUT object with x-scal-s3-version-id header', () => { Bucket: bucketName, VersioningConfiguration: { Status: 'Enabled', - } + }, }; const sParams = { Bucket: bucketName, VersioningConfiguration: { Status: 'Suspended', - } + }, }; const params = { Bucket: bucketName, Key: objectName }; let objMDBefore; @@ -642,43 +790,62 @@ describe('PUT object with x-scal-s3-version-id header', () => { let versionsAfter; let vId; - async.series([ - next => s3.send(new PutBucketVersioningCommand(vParams)).then(() => next()), - next => s3.send(new PutObjectCommand(params)).then(() => next()), - next => s3.send(new PutObjectCommand(params)).then(res => { - vId = res.VersionId; - return next(); - }), - next => fakeMetadataArchive(bucketName, objectName, vId, archive, next), - next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => { - versionsBefore = clearRestoreStatus(res.Versions); - return next(err); - }), - next => getMetadata(bucketName, objectName, vId, (err, objMD) => { - objMDBefore = objMD; - return next(err); - }), - next => s3.send(new PutBucketVersioningCommand(sParams)).then(() => next()).catch(next), - next => putObjectVersion(s3, params, vId, next), - next => getMetadata(bucketName, objectName, vId, (err, objMD) => { - objMDAfter = objMD; - return next(err); - }), - next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => { - versionsAfter = clearRestoreStatus(res.Versions); - return next(err); - }), - ], err => { - assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); - - checkVersionsAndUpdate(versionsBefore, versionsAfter, [0]); - assert.deepStrictEqual(versionsAfter, versionsBefore); - - checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'originOp', - 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName']); - assert.deepStrictEqual(objMDAfter, objMDBefore); - return done(); - }); + async.series( + [ + next => s3.send(new PutBucketVersioningCommand(vParams)).then(() => next()), + next => s3.send(new PutObjectCommand(params)).then(() => next()), + next => + s3.send(new PutObjectCommand(params)).then(res => { + vId = res.VersionId; + return next(); + }), + next => fakeMetadataArchive(bucketName, objectName, vId, archive, next), + next => + metadata.listObject(bucketName, mdListingParams, log, (err, res) => { + versionsBefore = clearRestoreStatus(res.Versions); + return next(err); + }), + next => + getMetadata(bucketName, objectName, vId, (err, objMD) => { + objMDBefore = objMD; + return next(err); + }), + next => + s3 + .send(new PutBucketVersioningCommand(sParams)) + .then(() => next()) + .catch(next), + next => putObjectVersion(s3, params, vId, next), + next => + getMetadata(bucketName, objectName, vId, (err, objMD) => { + objMDAfter = objMD; + return next(err); + }), + next => + metadata.listObject(bucketName, mdListingParams, log, (err, res) => { + versionsAfter = clearRestoreStatus(res.Versions); + return next(err); + }), + ], + err => { + assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); + + checkVersionsAndUpdate(versionsBefore, versionsAfter, [0]); + assert.deepStrictEqual(versionsAfter, versionsBefore); + + checkObjMdAndUpdate(objMDBefore, objMDAfter, [ + 'location', + 'content-length', + 'originOp', + 'microVersionId', + 'x-amz-restore', + 'archive', + 'dataStoreName', + ]); + assert.deepStrictEqual(objMDAfter, objMDBefore); + return done(); + }, + ); }); it('should overwrite the current null version after bucket version enabled', done => { @@ -686,7 +853,7 @@ describe('PUT object with x-scal-s3-version-id header', () => { Bucket: bucketName, VersioningConfiguration: { Status: 'Enabled', - } + }, }; const params = { Bucket: bucketName, Key: objectName }; let objMDBefore; @@ -694,38 +861,52 @@ describe('PUT object with x-scal-s3-version-id header', () => { let versionsBefore; let versionsAfter; - async.series([ - next => s3.send(new PutObjectCommand(params)).then(() => next()), - next => fakeMetadataArchive(bucketName, objectName, undefined, archive, next), - next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => { - versionsBefore = clearRestoreStatus(res.Versions); - return next(err); - }), - next => getMetadata(bucketName, objectName, undefined, (err, objMD) => { - objMDBefore = objMD; - return next(err); - }), - next => s3.send(new PutBucketVersioningCommand(vParams)).then(() => next()), - next => putObjectVersion(s3, params, 'null', next), - next => getMetadata(bucketName, objectName, undefined, (err, objMD) => { - objMDAfter = objMD; - return next(err); - }), - next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => { - versionsAfter = clearRestoreStatus(res.Versions); - return next(err); - }), - ], err => { - assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); - - checkVersionsAndUpdate(versionsBefore, versionsAfter, [0]); - assert.deepStrictEqual(versionsAfter, versionsBefore); - - checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'originOp', - 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName']); - assert.deepStrictEqual(objMDAfter, objMDBefore); - return done(); - }); + async.series( + [ + next => s3.send(new PutObjectCommand(params)).then(() => next()), + next => fakeMetadataArchive(bucketName, objectName, undefined, archive, next), + next => + metadata.listObject(bucketName, mdListingParams, log, (err, res) => { + versionsBefore = clearRestoreStatus(res.Versions); + return next(err); + }), + next => + getMetadata(bucketName, objectName, undefined, (err, objMD) => { + objMDBefore = objMD; + return next(err); + }), + next => s3.send(new PutBucketVersioningCommand(vParams)).then(() => next()), + next => putObjectVersion(s3, params, 'null', next), + next => + getMetadata(bucketName, objectName, undefined, (err, objMD) => { + objMDAfter = objMD; + return next(err); + }), + next => + metadata.listObject(bucketName, mdListingParams, log, (err, res) => { + versionsAfter = clearRestoreStatus(res.Versions); + return next(err); + }), + ], + err => { + assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); + + checkVersionsAndUpdate(versionsBefore, versionsAfter, [0]); + assert.deepStrictEqual(versionsAfter, versionsBefore); + + checkObjMdAndUpdate(objMDBefore, objMDAfter, [ + 'location', + 'content-length', + 'originOp', + 'microVersionId', + 'x-amz-restore', + 'archive', + 'dataStoreName', + ]); + assert.deepStrictEqual(objMDAfter, objMDBefore); + return done(); + }, + ); }); it('should fail if restore is already completed', done => { @@ -735,255 +916,299 @@ describe('PUT object with x-scal-s3-version-id header', () => { restoreRequestedAt: new Date(0), restoreRequestedDays: 5, restoreCompletedAt: new Date(10), - restoreWillExpireAt: new Date(10 + (5 * 24 * 60 * 60 * 1000)), + restoreWillExpireAt: new Date(10 + 5 * 24 * 60 * 60 * 1000), }; - async.series([ - next => s3.send(new PutObjectCommand(params)).then(() => next()), - next => fakeMetadataArchive(bucketName, objectName, undefined, archiveCompleted, next), - next => putObjectVersion(s3, params, '', err => { - checkError(err, 'InvalidObjectState', 403); - return next(); - }), - ], err => { - assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); - return done(); - }); + async.series( + [ + next => s3.send(new PutObjectCommand(params)).then(() => next()), + next => fakeMetadataArchive(bucketName, objectName, undefined, archiveCompleted, next), + next => + putObjectVersion(s3, params, '', err => { + checkError(err, 'InvalidObjectState', 403); + return next(); + }), + ], + err => { + assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); + return done(); + }, + ); }); - [ - 'non versioned', - 'versioned', - 'suspended' - ].forEach(versioning => { + ['non versioned', 'versioned', 'suspended'].forEach(versioning => { it(`should update restore metadata while keeping storage class (${versioning})`, done => { const params = { Bucket: bucketName, Key: objectName }; let objMDBefore; let objMDAfter; - - async.series([ - next => { - if (versioning === 'versioned') { - return s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: { Status: 'Enabled' } - })).then(() => next()); - } else if (versioning === 'suspended') { - return s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: { Status: 'Suspended' } - })).then(() => next()); - } - return next(); + async.series( + [ + next => { + if (versioning === 'versioned') { + return s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ) + .then(() => next()); + } else if (versioning === 'suspended') { + return s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: { Status: 'Suspended' }, + }), + ) + .then(() => next()); + } + return next(); + }, + next => s3.send(new PutObjectCommand(params)).then(() => next()), + next => fakeMetadataArchive(bucketName, objectName, undefined, archive, next), + next => + getMetadata(bucketName, objectName, undefined, (err, objMD) => { + objMDBefore = objMD; + return next(err); + }), + next => metadata.listObject(bucketName, mdListingParams, log, err => next(err)), + next => putObjectVersion(s3, params, '', next), + next => + getMetadata(bucketName, objectName, undefined, (err, objMD) => { + objMDAfter = objMD; + return next(err); + }), + next => + s3 + .send(new ListObjectsCommand({ Bucket: bucketName })) + .then(res => { + assert.strictEqual(res.Contents.length, 1); + assert.strictEqual(res.Contents[0].StorageClass, LOCATION_NAME_DMF); + return next(); + }) + .catch(err => { + assert.ifError(err); + return next(err); + }), + next => + s3 + .send(new HeadObjectCommand(params)) + .then(res => { + assert.strictEqual(res.StorageClass, LOCATION_NAME_DMF); + return next(); + }) + .catch(err => { + assert.ifError(err); + return next(err); + }), + next => + s3 + .send(new GetObjectCommand(params)) + .then(res => { + assert.strictEqual(res.StorageClass, LOCATION_NAME_DMF); + return next(); + }) + .catch(err => { + assert.ifError(err); + return next(err); + }), + ], + err => { + assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); + + // storage class must stay as the cold location + assert.deepStrictEqual(objMDAfter['x-amz-storage-class'], LOCATION_NAME_DMF); + + /// Make sure object data location is set back to its bucket data location. + assert.deepStrictEqual(objMDAfter.dataStoreName, 'us-east-1'); + + assert.deepStrictEqual(objMDAfter.archive.archiveInfo, objMDBefore.archive.archiveInfo); + assert.deepStrictEqual( + objMDAfter.archive.restoreRequestedAt, + objMDBefore.archive.restoreRequestedAt, + ); + assert.deepStrictEqual( + objMDAfter.archive.restoreRequestedDays, + objMDBefore.archive.restoreRequestedDays, + ); + assert.deepStrictEqual(objMDAfter['x-amz-restore']['ongoing-request'], false); + + assert(objMDAfter.archive.restoreCompletedAt); + assert(objMDAfter.archive.restoreWillExpireAt); + assert(objMDAfter['x-amz-restore']['expiry-date']); + return done(); }, - next => s3.send(new PutObjectCommand(params)).then(() => next()), - next => fakeMetadataArchive(bucketName, objectName, undefined, archive, next), - next => getMetadata(bucketName, objectName, undefined, (err, objMD) => { - objMDBefore = objMD; - return next(err); - }), - next => metadata.listObject(bucketName, mdListingParams, log, err => next(err)), - next => putObjectVersion(s3, params, '', next), - next => getMetadata(bucketName, objectName, undefined, (err, objMD) => { - objMDAfter = objMD; - return next(err); - }), - next => s3.send(new ListObjectsCommand({ Bucket: bucketName })).then(res => { - assert.strictEqual(res.Contents.length, 1); - assert.strictEqual(res.Contents[0].StorageClass, LOCATION_NAME_DMF); - return next(); - }).catch(err => { - assert.ifError(err); - return next(err); - }), - next => s3.send(new HeadObjectCommand(params)).then(res => { - assert.strictEqual(res.StorageClass, LOCATION_NAME_DMF); - return next(); - }).catch(err => { - assert.ifError(err); - return next(err); - }), - next => s3.send(new GetObjectCommand(params)).then(res => { - assert.strictEqual(res.StorageClass, LOCATION_NAME_DMF); - return next(); - }).catch(err => { - assert.ifError(err); - return next(err); - }), - ], err => { - assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); - - // storage class must stay as the cold location - assert.deepStrictEqual(objMDAfter['x-amz-storage-class'], LOCATION_NAME_DMF); - - /// Make sure object data location is set back to its bucket data location. - assert.deepStrictEqual(objMDAfter.dataStoreName, 'us-east-1'); - - assert.deepStrictEqual(objMDAfter.archive.archiveInfo, objMDBefore.archive.archiveInfo); - assert.deepStrictEqual(objMDAfter.archive.restoreRequestedAt, - objMDBefore.archive.restoreRequestedAt); - assert.deepStrictEqual(objMDAfter.archive.restoreRequestedDays, - objMDBefore.archive.restoreRequestedDays); - assert.deepStrictEqual(objMDAfter['x-amz-restore']['ongoing-request'], false); - - assert(objMDAfter.archive.restoreCompletedAt); - assert(objMDAfter.archive.restoreWillExpireAt); - assert(objMDAfter['x-amz-restore']['expiry-date']); - return done(); - }); + ); }); }); it('should "copy" all but non data-related metadata (data encryption, data size...)', done => { - const params = { - Bucket: bucketNameMD, - Key: objectName - }; - const putParams = { - ...params, - Metadata: { - 'custom-user-md': 'custom-md', - }, - WebsiteRedirectLocation: 'http://custom-redirect' - }; - const aclParams = { - ...params, - // email of user Bart defined in authdata.json - GrantFullControl: 'emailaddress=sampleaccount1@sampling.com', - }; - const tagParams = { - ...params, - Tagging: { - TagSet: [{ + const params = { + Bucket: bucketNameMD, + Key: objectName, + }; + const putParams = { + ...params, + Metadata: { + 'custom-user-md': 'custom-md', + }, + WebsiteRedirectLocation: 'http://custom-redirect', + }; + const aclParams = { + ...params, + // email of user Bart defined in authdata.json + GrantFullControl: 'emailaddress=sampleaccount1@sampling.com', + }; + const tagParams = { + ...params, + Tagging: { + TagSet: [ + { Key: 'tag1', - Value: 'value1' - }, { + Value: 'value1', + }, + { Key: 'tag2', - Value: 'value2' - }] - } - }; - const legalHoldParams = { - ...params, - LegalHold: { - Status: 'ON' + Value: 'value2', }, - }; - const acl = { - 'Canned': '', - 'FULL_CONTROL': [ - // canonicalID of user Bart - '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be', - ], - 'WRITE_ACP': [], - 'READ': [], - 'READ_ACP': [], - }; - const tags = { tag1: 'value1', tag2: 'value2' }; - const replicationInfo = { - 'status': 'COMPLETED', - 'backends': [ - { - 'site': 'azure-normal', - 'status': 'COMPLETED', - 'dataStoreVersionId': '', - }, ], - 'content': [ - 'DATA', - 'METADATA', - ], - 'destination': 'arn:aws:s3:::versioned', - 'storageClass': 'azure-normal', - 'role': 'arn:aws:iam::root:role/s3-replication-role', - 'storageType': 'azure', - 'dataStoreVersionId': '', - 'isNFS': null, + }, + }; + const legalHoldParams = { + ...params, + LegalHold: { + Status: 'ON', + }, + }; + const acl = { + Canned: '', + FULL_CONTROL: [ + // canonicalID of user Bart + '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be', + ], + WRITE_ACP: [], + READ: [], + READ_ACP: [], + }; + const tags = { tag1: 'value1', tag2: 'value2' }; + const replicationInfo = { + status: 'COMPLETED', + backends: [ + { + site: 'azure-normal', + status: 'COMPLETED', + dataStoreVersionId: '', + }, + ], + content: ['DATA', 'METADATA'], + destination: 'arn:aws:s3:::versioned', + storageClass: 'azure-normal', + role: 'arn:aws:iam::root:role/s3-replication-role', + storageType: 'azure', + dataStoreVersionId: '', + isNFS: null, }; - async.series([ - next => s3.send(new PutObjectCommand(putParams)).then(() => next()), - next => s3.send(new PutObjectAclCommand(aclParams)).then(() => next()), - next => s3.send(new PutObjectTaggingCommand(tagParams)).then(() => next()), - next => s3.send(new PutObjectLegalHoldCommand(legalHoldParams)).then(() => next()), - next => getMetadata(bucketNameMD, objectName, undefined, (err, objMD) => { - if (err) { - return next(err); - } - /* eslint-disable no-param-reassign */ - objMD.dataStoreName = LOCATION_NAME_DMF; - objMD.archive = archive; - objMD.replicationInfo = replicationInfo; - // data related - objMD['content-length'] = 99; - objMD['content-type'] = 'testtype'; - objMD['content-md5'] = 'testmd5'; - objMD['content-encoding'] = 'testencoding'; - objMD['x-amz-server-side-encryption'] = 'aws:kms'; - /* eslint-enable no-param-reassign */ - return metadata.putObjectMD(bucketNameMD, objectName, objMD, undefined, log, next); - }), - next => putObjectVersion(s3, params, '', next), - next => getMetadata(bucketNameMD, objectName, undefined, (err, objMD) => { - if (err) { - return next(err); - } - assert.deepStrictEqual(objMD.acl, acl); - assert.deepStrictEqual(objMD.tags, tags); - assert.deepStrictEqual(objMD.replicationInfo, replicationInfo); - assert.deepStrictEqual(objMD.legalHold, true); - assert.strictEqual(objMD['x-amz-meta-custom-user-md'], 'custom-md'); - assert.strictEqual(objMD['x-amz-website-redirect-location'], 'http://custom-redirect'); - // make sure data related metadatas ar not the same before and after - assert.notStrictEqual(objMD['x-amz-server-side-encryption'], 'aws:kms'); - assert.notStrictEqual(objMD['content-length'], 99); - assert.notStrictEqual(objMD['content-encoding'], 'testencoding'); - assert.notStrictEqual(objMD['content-type'], 'testtype'); - // make sure we keep the same etag and add the new restored - // data's etag inside x-amz-restore - assert.strictEqual(objMD['content-md5'], 'testmd5'); - assert.strictEqual(typeof objMD['x-amz-restore']['content-md5'], 'string'); - return next(); - }), - // removing legal hold to be able to clean the bucket after the test - next => { - legalHoldParams.LegalHold.Status = 'OFF'; - return s3.send(new PutObjectLegalHoldCommand(legalHoldParams)).then(() => next()); - }, - ], done); + async.series( + [ + next => s3.send(new PutObjectCommand(putParams)).then(() => next()), + next => s3.send(new PutObjectAclCommand(aclParams)).then(() => next()), + next => s3.send(new PutObjectTaggingCommand(tagParams)).then(() => next()), + next => s3.send(new PutObjectLegalHoldCommand(legalHoldParams)).then(() => next()), + next => + getMetadata(bucketNameMD, objectName, undefined, (err, objMD) => { + if (err) { + return next(err); + } + /* eslint-disable no-param-reassign */ + objMD.dataStoreName = LOCATION_NAME_DMF; + objMD.archive = archive; + objMD.replicationInfo = replicationInfo; + // data related + objMD['content-length'] = 99; + objMD['content-type'] = 'testtype'; + objMD['content-md5'] = 'testmd5'; + objMD['content-encoding'] = 'testencoding'; + objMD['x-amz-server-side-encryption'] = 'aws:kms'; + /* eslint-enable no-param-reassign */ + return metadata.putObjectMD(bucketNameMD, objectName, objMD, undefined, log, next); + }), + next => putObjectVersion(s3, params, '', next), + next => + getMetadata(bucketNameMD, objectName, undefined, (err, objMD) => { + if (err) { + return next(err); + } + assert.deepStrictEqual(objMD.acl, acl); + assert.deepStrictEqual(objMD.tags, tags); + assert.deepStrictEqual(objMD.replicationInfo, replicationInfo); + assert.deepStrictEqual(objMD.legalHold, true); + assert.strictEqual(objMD['x-amz-meta-custom-user-md'], 'custom-md'); + assert.strictEqual(objMD['x-amz-website-redirect-location'], 'http://custom-redirect'); + // make sure data related metadatas ar not the same before and after + assert.notStrictEqual(objMD['x-amz-server-side-encryption'], 'aws:kms'); + assert.notStrictEqual(objMD['content-length'], 99); + assert.notStrictEqual(objMD['content-encoding'], 'testencoding'); + assert.notStrictEqual(objMD['content-type'], 'testtype'); + // make sure we keep the same etag and add the new restored + // data's etag inside x-amz-restore + assert.strictEqual(objMD['content-md5'], 'testmd5'); + assert.strictEqual(typeof objMD['x-amz-restore']['content-md5'], 'string'); + return next(); + }), + // removing legal hold to be able to clean the bucket after the test + next => { + legalHoldParams.LegalHold.Status = 'OFF'; + return s3.send(new PutObjectLegalHoldCommand(legalHoldParams)).then(() => next()); + }, + ], + done, + ); }); it('should set restore originOp and drop restore-attempt metadata', done => { const params = { Bucket: bucketName, Key: objectName }; - async.series([ - next => s3.send(new PutObjectCommand({ - ...params, - Metadata: { - 'custom-md': 'preserved-value', - }, - })).then(() => next()).catch(next), - next => fakeMetadataArchive(bucketName, objectName, undefined, archive, next), - next => getMetadata(bucketName, objectName, undefined, (err, objMD) => { - if (err) { - return next(err); - } - /* eslint-disable no-param-reassign */ - objMD['x-amz-meta-scal-s3-restore-attempt'] = '3'; - /* eslint-enable no-param-reassign */ - return metadata.putObjectMD(bucketName, objectName, objMD, undefined, log, next); - }), - next => putObjectVersion(s3, params, '', next), - next => getMetadata(bucketName, objectName, undefined, (err, objMD) => { - if (err) { - return next(err); - } - assert.strictEqual(objMD.originOp, 's3:ObjectRestore:Completed'); - assert.strictEqual(objMD['x-amz-meta-custom-md'], 'preserved-value'); - assert.strictEqual(objMD['x-amz-meta-scal-s3-restore-attempt'], undefined); - return next(); - }), - ], done); + async.series( + [ + next => + s3 + .send( + new PutObjectCommand({ + ...params, + Metadata: { + 'custom-md': 'preserved-value', + }, + }), + ) + .then(() => next()) + .catch(next), + next => fakeMetadataArchive(bucketName, objectName, undefined, archive, next), + next => + getMetadata(bucketName, objectName, undefined, (err, objMD) => { + if (err) { + return next(err); + } + /* eslint-disable no-param-reassign */ + objMD['x-amz-meta-scal-s3-restore-attempt'] = '3'; + /* eslint-enable no-param-reassign */ + return metadata.putObjectMD(bucketName, objectName, objMD, undefined, log, next); + }), + next => putObjectVersion(s3, params, '', next), + next => + getMetadata(bucketName, objectName, undefined, (err, objMD) => { + if (err) { + return next(err); + } + assert.strictEqual(objMD.originOp, 's3:ObjectRestore:Completed'); + assert.strictEqual(objMD['x-amz-meta-custom-md'], 'preserved-value'); + assert.strictEqual(objMD['x-amz-meta-scal-s3-restore-attempt'], undefined); + return next(); + }), + ], + done, + ); }); it('should keep x-amz-meta-scal-version-id when restoring on ingestion bucket', async () => { @@ -991,12 +1216,14 @@ describe('PUT object with x-scal-s3-version-id header', () => { const params = { Bucket: ingestionBucketName, Key: objectName }; let putVersionId; try { - await s3.send(new CreateBucketCommand({ - Bucket: ingestionBucketName, - CreateBucketConfiguration: { - LocationConstraint: 'us-east-2:ingest', - }, - })); + await s3.send( + new CreateBucketCommand({ + Bucket: ingestionBucketName, + CreateBucketConfiguration: { + LocationConstraint: 'us-east-2:ingest', + }, + }), + ); const putRes = await s3.send(new PutObjectCommand(params)); putVersionId = putRes.VersionId; @@ -1005,8 +1232,7 @@ describe('PUT object with x-scal-s3-version-id header', () => { await putObjectVersion(s3, params, putVersionId); - const restoredObjMD = await getMetadata( - ingestionBucketName, objectName, putVersionId); + const restoredObjMD = await getMetadata(ingestionBucketName, objectName, putVersionId); assert.strictEqual(restoredObjMD['x-amz-meta-scal-version-id'], putVersionId); } finally { diff --git a/tests/functional/aws-node-sdk/test/object/rangeTest.js b/tests/functional/aws-node-sdk/test/object/rangeTest.js index 3e0a66b606..fa5a671aa1 100644 --- a/tests/functional/aws-node-sdk/test/object/rangeTest.js +++ b/tests/functional/aws-node-sdk/test/object/rangeTest.js @@ -32,8 +32,7 @@ function getOuterRange(range, bytes) { arr[1] = Number.parseInt(bytes, 10) - 1; } else { arr[0] = arr[0] === '' ? 0 : Number.parseInt(arr[0], 10); - arr[1] = arr[1] === '' || Number.parseInt(arr[1], 10) >= bytes ? - Number.parseInt(bytes, 10) - 1 : arr[1]; + arr[1] = arr[1] === '' || Number.parseInt(arr[1], 10) >= bytes ? Number.parseInt(bytes, 10) - 1 : arr[1]; } return { begin: arr[0], @@ -44,58 +43,69 @@ function getOuterRange(range, bytes) { // Get the ranged object from a bucket. Write the response body to a file, then // use getRangeExec to check that all the bytes are in the correct location. function checkRanges(range, bytes) { - return s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - Range: `bytes=${range}`, - })) - .then(async res => { - const { begin, end } = getOuterRange(range, bytes); - const total = (end - begin) + 1; - // If the range header is '-' (i.e., it is invalid), content range - // should be undefined - const contentRange = range === '-' ? undefined : - `bytes ${begin}-${end}/${bytes}`; + return s3 + .send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + Range: `bytes=${range}`, + }), + ) + .then(async res => { + const { begin, end } = getOuterRange(range, bytes); + const total = end - begin + 1; + // If the range header is '-' (i.e., it is invalid), content range + // should be undefined + const contentRange = range === '-' ? undefined : `bytes ${begin}-${end}/${bytes}`; - assert.deepStrictEqual(res.ContentLength, total); - assert.deepStrictEqual(res.ContentRange, contentRange); - assert(res.ContentType === undefined || - res.ContentType === 'application/octet-stream'); - assert.deepStrictEqual(res.Metadata, {}); + assert.deepStrictEqual(res.ContentLength, total); + assert.deepStrictEqual(res.ContentRange, contentRange); + assert(res.ContentType === undefined || res.ContentType === 'application/octet-stream'); + assert.deepStrictEqual(res.Metadata, {}); - const bodyBytes = await res.Body.transformToByteArray(); - const bodyBuffer = Buffer.from(bodyBytes); - await writeFileAsync(`hashedFile.${bytes}.${range}`, bodyBuffer); - return execFileAsync('./getRangeExec', ['--check', '--size', total, - '--offset', begin, `hashedFile.${bytes}.${range}`]); - }); + const bodyBytes = await res.Body.transformToByteArray(); + const bodyBuffer = Buffer.from(bodyBytes); + await writeFileAsync(`hashedFile.${bytes}.${range}`, bodyBuffer); + return execFileAsync('./getRangeExec', [ + '--check', + '--size', + total, + '--offset', + begin, + `hashedFile.${bytes}.${range}`, + ]); + }); } // Create 5MB parts and upload them as parts of a MPU. Returns array of part // responses (with ETag) for CompleteMultipartUpload. async function uploadParts(bytes, uploadId) { const name = `hashedFile.${bytes}`; - return Promise.all([1, 2].map(async part => { - try { - await execFileAsync('dd', [ - `if=${name}`, - `of=${name}.mpuPart${part}`, - 'bs=5242880', - `skip=${part - 1}`, - 'count=1', - ]); - const res = await s3.send(new UploadPartCommand({ - Bucket: bucket, - Key: key, - PartNumber: part, - UploadId: uploadId, - Body: createReadStream(`${name}.mpuPart${part}`), - })); - return res; - } catch (error) { - throw new Error(`Error uploading part ${part}: ${error.message}`); - } - })); + return Promise.all( + [1, 2].map(async part => { + try { + await execFileAsync('dd', [ + `if=${name}`, + `of=${name}.mpuPart${part}`, + 'bs=5242880', + `skip=${part - 1}`, + 'count=1', + ]); + const res = await s3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: key, + PartNumber: part, + UploadId: uploadId, + Body: createReadStream(`${name}.mpuPart${part}`), + }), + ); + return res; + } catch (error) { + throw new Error(`Error uploading part ${part}: ${error.message}`); + } + }), + ); } // Create a hashed file of size bytes @@ -105,8 +115,7 @@ function createHashedFile(bytes) { } describe('aws-node-sdk range tests', () => { - before(() => execFileAsync('gcc', ['-o', 'getRangeExec', - 'lib/utility/getRange.c'])); + before(() => execFileAsync('gcc', ['-o', 'getRangeExec', 'lib/utility/getRange.c'])); after(() => execAsync('rm getRangeExec')); describe('aws-node-sdk range test for object put by MPU', () => @@ -117,65 +126,79 @@ describe('aws-node-sdk range tests', () => { let uploadId; beforeEach(() => - s3.send(new CreateBucketCommand({ Bucket: bucket })) - .then(() => s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: key, - }))) - .then(res => { - uploadId = res.UploadId; - }) - .then(() => createHashedFile(fileSize)) - .then(() => uploadParts(fileSize, uploadId)) - .then(res => s3.send(new CompleteMultipartUploadCommand({ - Bucket: bucket, - Key: key, - UploadId: uploadId, - MultipartUpload: { - Parts: [ - { - ETag: res[0].ETag, - PartNumber: 1, - }, - { - ETag: res[1].ETag, - PartNumber: 2, - }, - ], - }, - }))) + s3 + .send(new CreateBucketCommand({ Bucket: bucket })) + .then(() => + s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + }), + ), + ) + .then(res => { + uploadId = res.UploadId; + }) + .then(() => createHashedFile(fileSize)) + .then(() => uploadParts(fileSize, uploadId)) + .then(res => + s3.send( + new CompleteMultipartUploadCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + MultipartUpload: { + Parts: [ + { + ETag: res[0].ETag, + PartNumber: 1, + }, + { + ETag: res[1].ETag, + PartNumber: 2, + }, + ], + }, + }), + ), + ), ); - afterEach(() => bucketUtil.empty(bucket) - .then(() => s3.send(new AbortMultipartUploadCommand({ - Bucket: bucket, - Key: key, - UploadId: uploadId, - }))) - .catch(err => { - // Upload was already completed in beforeEach; abort is no-op - if (err.name === 'NoSuchUpload' || err.code === 'NoSuchUpload') { - return; - } - throw err; - }) - .then(() => bucketUtil.deleteOne(bucket)) - .then(() => execAsync(`rm hashedFile.${fileSize}*`)) + afterEach(() => + bucketUtil + .empty(bucket) + .then(() => + s3.send( + new AbortMultipartUploadCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + }), + ), + ) + .catch(err => { + // Upload was already completed in beforeEach; abort is no-op + if (err.name === 'NoSuchUpload' || err.code === 'NoSuchUpload') { + return; + } + throw err; + }) + .then(() => bucketUtil.deleteOne(bucket)) + .then(() => execAsync(`rm hashedFile.${fileSize}*`)), ); - it('should get a range from the first part of an object', () => - checkRanges('0-9', fileSize)); + it('should get a range from the first part of an object', () => checkRanges('0-9', fileSize)); - it('should get a range from the second part of an object', () => - checkRanges('5242880-5242889', fileSize)); + it('should get a range from the second part of an object', () => checkRanges('5242880-5242889', fileSize)); - it('should get a range that spans both parts of an object', () => - checkRanges('5242875-5242884', fileSize)); + it('should get a range that spans both parts of an object', () => checkRanges('5242875-5242884', fileSize)); - it('should get a range from the second part of an object and ' + - 'include the end if the range requested goes beyond the ' + - 'actual object end', () => - checkRanges('10485750-10485790', fileSize)); + it( + 'should get a range from the second part of an object and ' + + 'include the end if the range requested goes beyond the ' + + 'actual object end', + () => checkRanges('10485750-10485790', fileSize), + ); })); describe('aws-node-sdk range test of regular object put (non-MPU)', () => @@ -185,18 +208,26 @@ describe('aws-node-sdk range tests', () => { const fileSize = 2000; beforeEach(() => - s3.send(new CreateBucketCommand({ Bucket: bucket })) - .then(() => createHashedFile(fileSize)) - .then(() => s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: createReadStream(`hashedFile.${fileSize}`), - })))); + s3 + .send(new CreateBucketCommand({ Bucket: bucket })) + .then(() => createHashedFile(fileSize)) + .then(() => + s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: createReadStream(`hashedFile.${fileSize}`), + }), + ), + ), + ); afterEach(() => - bucketUtil.empty(bucket) - .then(() => bucketUtil.deleteOne(bucket)) - .then(() => execAsync(`rm hashedFile.${fileSize}*`))); + bucketUtil + .empty(bucket) + .then(() => bucketUtil.deleteOne(bucket)) + .then(() => execAsync(`rm hashedFile.${fileSize}*`)), + ); const putRangeTests = [ '-', // Test for invalid range @@ -229,9 +260,9 @@ describe('aws-node-sdk range tests', () => { ]; putRangeTests.forEach(range => { - it(`should get a range of ${range} bytes using a ${fileSize} ` + - 'byte sized object', () => - checkRanges(range, fileSize)); + it(`should get a range of ${range} bytes using a ${fileSize} ` + 'byte sized object', () => + checkRanges(range, fileSize), + ); }); })); @@ -242,26 +273,36 @@ describe('aws-node-sdk range tests', () => { const fileSize = 2900; beforeEach(() => - s3.send(new CreateBucketCommand({ Bucket: bucket })) - .then(() => createHashedFile(fileSize)) - .then(() => s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: createReadStream(`hashedFile.${fileSize}`), - })))); + s3 + .send(new CreateBucketCommand({ Bucket: bucket })) + .then(() => createHashedFile(fileSize)) + .then(() => + s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: createReadStream(`hashedFile.${fileSize}`), + }), + ), + ), + ); afterEach(() => - bucketUtil.empty(bucket) - .then(() => bucketUtil.deleteOne(bucket)) - .then(() => execAsync(`rm hashedFile.${fileSize}*`))); + bucketUtil + .empty(bucket) + .then(() => bucketUtil.deleteOne(bucket)) + .then(() => execAsync(`rm hashedFile.${fileSize}*`)), + ); - it('should get the final 90 bytes of a 2890 byte object for a ' + - 'byte range of 2800-', () => - checkRanges('2800-', fileSize)); + it('should get the final 90 bytes of a 2890 byte object for a ' + 'byte range of 2800-', () => + checkRanges('2800-', fileSize), + ); - it('should get the final 90 bytes of a 2890 byte object for a ' + - 'byte range of 2800-Number.MAX_SAFE_INTEGER', () => - checkRanges(`2800-${Number.MAX_SAFE_INTEGER}`, fileSize)); + it( + 'should get the final 90 bytes of a 2890 byte object for a ' + + 'byte range of 2800-Number.MAX_SAFE_INTEGER', + () => checkRanges(`2800-${Number.MAX_SAFE_INTEGER}`, fileSize), + ); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/object/websiteFiles/error.html b/tests/functional/aws-node-sdk/test/object/websiteFiles/error.html index 71f5a25838..5c73b35224 100644 --- a/tests/functional/aws-node-sdk/test/object/websiteFiles/error.html +++ b/tests/functional/aws-node-sdk/test/object/websiteFiles/error.html @@ -1,9 +1,9 @@ - - Error!! - - -

It appears you messed up

-

Or maybe it was me...

- + + Error!! + + +

It appears you messed up

+

Or maybe it was me...

+ diff --git a/tests/functional/aws-node-sdk/test/object/websiteFiles/index.html b/tests/functional/aws-node-sdk/test/object/websiteFiles/index.html index 8ce654d9a3..832b3d787a 100644 --- a/tests/functional/aws-node-sdk/test/object/websiteFiles/index.html +++ b/tests/functional/aws-node-sdk/test/object/websiteFiles/index.html @@ -1,10 +1,11 @@ - - Best testing website ever - - -

Welcome to my extraordinary bucket website testing page

-

Now hosted on Scality's S3 Server -- a symphonic storage experience!

-
- + + Best testing website ever + + +

Welcome to my extraordinary bucket website testing page

+

Now hosted on Scality's S3 Server -- a symphonic storage experience!

+
+ + diff --git a/tests/functional/aws-node-sdk/test/object/websiteFiles/redirect.html b/tests/functional/aws-node-sdk/test/object/websiteFiles/redirect.html index 1f02b665d9..b16f62eebd 100644 --- a/tests/functional/aws-node-sdk/test/object/websiteFiles/redirect.html +++ b/tests/functional/aws-node-sdk/test/object/websiteFiles/redirect.html @@ -1,8 +1,8 @@ - - Best redirect link ever - - -

Welcome to your redirection file

- + + Best redirect link ever + + +

Welcome to your redirection file

+ diff --git a/tests/functional/aws-node-sdk/test/object/websiteGet.js b/tests/functional/aws-node-sdk/test/object/websiteGet.js index e66144385f..39a3db6eb3 100644 --- a/tests/functional/aws-node-sdk/test/object/websiteGet.js +++ b/tests/functional/aws-node-sdk/test/object/websiteGet.js @@ -22,32 +22,49 @@ const config = getConfig('default'); const s3Client = new S3Client(config); const s3 = { createBucket: (params, cb) => { - s3Client.send(new CreateBucketCommand(params)).then(d => cb(null, d)).catch(cb); + s3Client + .send(new CreateBucketCommand(params)) + .then(d => cb(null, d)) + .catch(cb); }, deleteBucket: (params, cb) => { - s3Client.send(new DeleteBucketCommand(params)).then(d => cb(null, d)).catch(cb); + s3Client + .send(new DeleteBucketCommand(params)) + .then(d => cb(null, d)) + .catch(cb); }, putBucketWebsite: (params, cb) => { - s3Client.send(new PutBucketWebsiteCommand(params)).then(d => cb(null, d)).catch(cb); + s3Client + .send(new PutBucketWebsiteCommand(params)) + .then(d => cb(null, d)) + .catch(cb); }, putObject: (params, cb) => { - s3Client.send(new PutObjectCommand(params)).then(d => cb(null, d)).catch(cb); + s3Client + .send(new PutObjectCommand(params)) + .then(d => cb(null, d)) + .catch(cb); }, deleteObject: (params, cb) => { - s3Client.send(new DeleteObjectCommand(params)).then(d => cb(null, d)).catch(cb); + s3Client + .send(new DeleteObjectCommand(params)) + .then(d => cb(null, d)) + .catch(cb); }, putBucketPolicy: (params, cb) => { - s3Client.send(new PutBucketPolicyCommand(params)).then(d => cb(null, d)).catch(cb); + s3Client + .send(new PutBucketPolicyCommand(params)) + .then(d => cb(null, d)) + .catch(cb); }, }; const transport = conf.https ? 'https' : 'http'; -const bucket = process.env.AWS_ON_AIR ? 'awsbucketwebsitetester' : - 'bucketwebsitetester'; +const bucket = process.env.AWS_ON_AIR ? 'awsbucketwebsitetester' : 'bucketwebsitetester'; const port = process.env.AWS_ON_AIR ? 80 : 8000; -const hostname = process.env.S3_END_TO_END ? - `${bucket}.s3-website-us-east-1.scality.com` : - `${bucket}.s3-website-us-east-1.amazonaws.com`; +const hostname = process.env.S3_END_TO_END + ? `${bucket}.s3-website-us-east-1.scality.com` + : `${bucket}.s3-website-us-east-1.amazonaws.com`; const endpoint = `${transport}://${hostname}:${port}`; const redirectEndpoint = `${transport}://www.google.com`; @@ -58,27 +75,33 @@ const redirectEndpoint = `${transport}://www.google.com`; function putBucketWebsiteAndPutObjectRedirect(redirect, condition, key, done) { const webConfig = new WebsiteConfigTester('index.html'); webConfig.addRoutingRule(redirect, condition); - s3.putBucketWebsite({ Bucket: bucket, - WebsiteConfiguration: webConfig }, err => { + s3.putBucketWebsite({ Bucket: bucket, WebsiteConfiguration: webConfig }, err => { if (err) { done(err); } - return s3.putObject({ Bucket: bucket, - Key: key, - ACL: 'public-read', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/redirect.html')), - ContentType: 'text/html' }, done); + return s3.putObject( + { + Bucket: bucket, + Key: key, + ACL: 'public-read', + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/redirect.html')), + ContentType: 'text/html', + }, + done, + ); }); } describe('User visits bucket website endpoint', () => { it('should return 404 when no such bucket', done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: endpoint, - responseType: '404-no-such-bucket', - }, done); + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: endpoint, + responseType: '404-no-such-bucket', + }, + done, + ); }); describe('with existing bucket', () => { @@ -87,122 +110,138 @@ describe('User visits bucket website endpoint', () => { afterEach(done => s3.deleteBucket({ Bucket: bucket }, done)); it('should return 404 when no website configuration', done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: endpoint, - responseType: '404-no-such-website-configuration', - }, done); + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: endpoint, + responseType: '404-no-such-website-configuration', + }, + done, + ); }); describe('with existing configuration', () => { beforeEach(done => { const webConfig = new WebsiteConfigTester('index.html'); - s3.putBucketWebsite({ Bucket: bucket, - WebsiteConfiguration: webConfig }, err => { - assert.strictEqual(err, - null, `Found unexpected err ${err}`); - s3.putObject({ Bucket: bucket, Key: 'index.html', - ACL: 'public-read', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/index.html')), - ContentType: 'text/html' }, + s3.putBucketWebsite({ Bucket: bucket, WebsiteConfiguration: webConfig }, err => { + assert.strictEqual(err, null, `Found unexpected err ${err}`); + s3.putObject( + { + Bucket: bucket, + Key: 'index.html', + ACL: 'public-read', + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/index.html')), + ContentType: 'text/html', + }, err => { assert.strictEqual(err, null); done(); - }); + }, + ); }); }); afterEach(done => { - s3.deleteObject({ Bucket: bucket, Key: 'index.html' }, - err => done(err)); - }); - - it('should return 405 when user requests method other than get ' + - 'or head', done => { - makeRequest({ - hostname, - port, - method: 'POST', - }, (err, res) => { - assert.strictEqual(err, null, - `Err with request ${err}`); - assert.strictEqual(res.statusCode, 405); - assert(res.body.indexOf('405 ' + - 'Method Not Allowed') > -1); - return done(); - }); + s3.deleteObject({ Bucket: bucket, Key: 'index.html' }, err => done(err)); + }); + + it('should return 405 when user requests method other than get ' + 'or head', done => { + makeRequest( + { + hostname, + port, + method: 'POST', + }, + (err, res) => { + assert.strictEqual(err, null, `Err with request ${err}`); + assert.strictEqual(res.statusCode, 405); + assert(res.body.indexOf('405 ' + 'Method Not Allowed') > -1); + return done(); + }, + ); }); it('should serve indexDocument if no key requested', done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: endpoint, - responseType: 'index-user', - }, done); + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: endpoint, + responseType: 'index-user', + }, + done, + ); }); it('should serve indexDocument if key requested', done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: `${endpoint}/index.html`, - responseType: 'index-user', - }, done); + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: `${endpoint}/index.html`, + responseType: 'index-user', + }, + done, + ); }); }); describe('with path in request with/without key', () => { beforeEach(done => { const webConfig = new WebsiteConfigTester('index.html'); - s3.putBucketWebsite({ Bucket: bucket, - WebsiteConfiguration: webConfig }, err => { - assert.strictEqual(err, - null, `Found unexpected err ${err}`); - s3.putObject({ Bucket: bucket, - Key: 'pathprefix/index.html', - ACL: 'public-read', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/index.html')), - ContentType: 'text/html' }, done); + s3.putBucketWebsite({ Bucket: bucket, WebsiteConfiguration: webConfig }, err => { + assert.strictEqual(err, null, `Found unexpected err ${err}`); + s3.putObject( + { + Bucket: bucket, + Key: 'pathprefix/index.html', + ACL: 'public-read', + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/index.html')), + ContentType: 'text/html', + }, + done, + ); }); }); afterEach(done => { - s3.deleteObject({ Bucket: bucket, Key: - 'pathprefix/index.html' }, - done); + s3.deleteObject({ Bucket: bucket, Key: 'pathprefix/index.html' }, done); }); - it('should serve indexDocument if path request without key', - done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: `${endpoint}/pathprefix/`, - responseType: 'index-user', - }, done); + it('should serve indexDocument if path request without key', done => { + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: `${endpoint}/pathprefix/`, + responseType: 'index-user', + }, + done, + ); }); - it('should serve indexDocument if path request with key', - done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: `${endpoint}/pathprefix/index.html`, - responseType: 'index-user', - }, done); + it('should serve indexDocument if path request with key', done => { + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: `${endpoint}/pathprefix/index.html`, + responseType: 'index-user', + }, + done, + ); }); }); describe('with private key', () => { beforeEach(done => { const webConfig = new WebsiteConfigTester('index.html'); - s3.putBucketWebsite({ Bucket: bucket, - WebsiteConfiguration: webConfig }, err => { - assert.strictEqual(err, - null, `Found unexpected err ${err}`); - s3.putObject({ Bucket: bucket, - Key: 'index.html', - ACL: 'private', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/index.html')), - ContentType: 'text/html' }, done); + s3.putBucketWebsite({ Bucket: bucket, WebsiteConfiguration: webConfig }, err => { + assert.strictEqual(err, null, `Found unexpected err ${err}`); + s3.putObject( + { + Bucket: bucket, + Key: 'index.html', + ACL: 'private', + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/index.html')), + ContentType: 'text/html', + }, + done, + ); }); }); @@ -211,27 +250,32 @@ describe('User visits bucket website endpoint', () => { }); it('should return 403 if key is private', done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: endpoint, - responseType: '403-access-denied', - }, done); + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: endpoint, + responseType: '403-access-denied', + }, + done, + ); }); }); describe('with nonexisting index document key', () => { beforeEach(done => { const webConfig = new WebsiteConfigTester('index.html'); - s3.putBucketWebsite({ Bucket: bucket, - WebsiteConfiguration: webConfig }, done); + s3.putBucketWebsite({ Bucket: bucket, WebsiteConfiguration: webConfig }, done); }); it('should return 403 if nonexisting index document key', done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: endpoint, - responseType: '403-access-denied', - }, done); + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: endpoint, + responseType: '403-access-denied', + }, + done, + ); }); }); @@ -240,80 +284,91 @@ describe('User visits bucket website endpoint', () => { const redirectAllTo = { HostName: 'www.google.com', }; - const webConfig = new WebsiteConfigTester(null, null, - redirectAllTo); - s3.putBucketWebsite({ Bucket: bucket, - WebsiteConfiguration: webConfig }, done); + const webConfig = new WebsiteConfigTester(null, null, redirectAllTo); + s3.putBucketWebsite({ Bucket: bucket, WebsiteConfiguration: webConfig }, done); }); it(`should redirect to ${redirectEndpoint}`, done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: endpoint, - responseType: 'redirect', - redirectUrl: `${redirectEndpoint}/`, - }, done); + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: endpoint, + responseType: 'redirect', + redirectUrl: `${redirectEndpoint}/`, + }, + done, + ); }); it(`should redirect to ${redirectEndpoint}/about`, done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: `${endpoint}/about`, - responseType: 'redirect', - redirectUrl: `${redirectEndpoint}/about`, - }, done); + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: `${endpoint}/about`, + responseType: 'redirect', + redirectUrl: `${redirectEndpoint}/about`, + }, + done, + ); }); }); - describe.skip('redirect all requests to https://www.google.com ' + - 'since https protocol set in website config', () => { - // Note: these tests will all redirect to https even if - // conf does not have https since protocol in website config - // specifies https - beforeEach(done => { - const redirectAllTo = { - HostName: 'www.google.com', - Protocol: 'https', - }; - const webConfig = new WebsiteConfigTester(null, null, - redirectAllTo); - s3.putBucketWebsite({ Bucket: bucket, - WebsiteConfiguration: webConfig }, done); - }); + describe.skip( + 'redirect all requests to https://www.google.com ' + 'since https protocol set in website config', + () => { + // Note: these tests will all redirect to https even if + // conf does not have https since protocol in website config + // specifies https + beforeEach(done => { + const redirectAllTo = { + HostName: 'www.google.com', + Protocol: 'https', + }; + const webConfig = new WebsiteConfigTester(null, null, redirectAllTo); + s3.putBucketWebsite({ Bucket: bucket, WebsiteConfiguration: webConfig }, done); + }); - it('should redirect to https://google.com/', done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: endpoint, - responseType: 'redirect', - redirectUrl: 'https://www.google.com/', - }, done); - }); + it('should redirect to https://google.com/', done => { + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: endpoint, + responseType: 'redirect', + redirectUrl: 'https://www.google.com/', + }, + done, + ); + }); - it('should redirect to https://google.com/about', done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: `${endpoint}/about`, - responseType: 'redirect', - redirectUrl: 'https://www.google.com/about', - }, done); - }); - }); + it('should redirect to https://google.com/about', done => { + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: `${endpoint}/about`, + responseType: 'redirect', + redirectUrl: 'https://www.google.com/about', + }, + done, + ); + }); + }, + ); describe('with custom error document', () => { beforeEach(done => { - const webConfig = new WebsiteConfigTester('index.html', - 'error.html'); - s3.putBucketWebsite({ Bucket: bucket, - WebsiteConfiguration: webConfig }, err => { - assert.strictEqual(err, - null, `Found unexpected err ${err}`); - s3.putObject({ Bucket: bucket, - Key: 'error.html', - ACL: 'public-read', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/error.html')), - ContentType: 'text/html' }, done); + const webConfig = new WebsiteConfigTester('index.html', 'error.html'); + s3.putBucketWebsite({ Bucket: bucket, WebsiteConfiguration: webConfig }, err => { + assert.strictEqual(err, null, `Found unexpected err ${err}`); + s3.putObject( + { + Bucket: bucket, + Key: 'error.html', + ACL: 'public-read', + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/error.html')), + ContentType: 'text/html', + }, + done, + ); }); }); @@ -321,55 +376,62 @@ describe('User visits bucket website endpoint', () => { s3.deleteObject({ Bucket: bucket, Key: 'error.html' }, done); }); - it('should serve custom error document if an error occurred', - done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: endpoint, - responseType: 'error-user', - }, done); - }); - - it('should serve custom error document with redirect', - done => { - s3.putObject({ Bucket: bucket, - Key: 'error.html', - ACL: 'public-read', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/error.html')), - ContentType: 'text/html', - WebsiteRedirectLocation: 'https://scality.com/test', - }, err => { - assert.ifError(err); - WebsiteConfigTester.checkHTML({ + it('should serve custom error document if an error occurred', done => { + WebsiteConfigTester.checkHTML( + { method: 'GET', url: endpoint, - responseType: 'redirect-error', - redirectUrl: 'https://scality.com/test', - expectedHeaders: { - 'x-amz-error-code': 'AccessDenied', - 'x-amz-error-message': 'Access Denied', - }, - }, done); - }); + responseType: 'error-user', + }, + done, + ); + }); + + it('should serve custom error document with redirect', done => { + s3.putObject( + { + Bucket: bucket, + Key: 'error.html', + ACL: 'public-read', + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/error.html')), + ContentType: 'text/html', + WebsiteRedirectLocation: 'https://scality.com/test', + }, + err => { + assert.ifError(err); + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: endpoint, + responseType: 'redirect-error', + redirectUrl: 'https://scality.com/test', + expectedHeaders: { + 'x-amz-error-code': 'AccessDenied', + 'x-amz-error-message': 'Access Denied', + }, + }, + done, + ); + }, + ); }); }); describe('unfound custom error document', () => { beforeEach(done => { - const webConfig = new WebsiteConfigTester('index.html', - 'error.html'); - s3.putBucketWebsite({ Bucket: bucket, - WebsiteConfiguration: webConfig }, done); + const webConfig = new WebsiteConfigTester('index.html', 'error.html'); + s3.putBucketWebsite({ Bucket: bucket, WebsiteConfiguration: webConfig }, done); }); - it('should serve s3 error file if unfound custom error document ' + - 'and an error occurred', done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: endpoint, - responseType: '403-retrieve-error-document', - }, done); + it('should serve s3 error file if unfound custom error document ' + 'and an error occurred', done => { + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: endpoint, + responseType: '403-retrieve-error-document', + }, + done, + ); }); }); @@ -383,18 +445,19 @@ describe('User visits bucket website endpoint', () => { HostName: 'www.google.com', }; webConfig.addRoutingRule(redirect, condition); - s3.putBucketWebsite({ Bucket: bucket, - WebsiteConfiguration: webConfig }, done); + s3.putBucketWebsite({ Bucket: bucket, WebsiteConfiguration: webConfig }, done); }); - it(`should redirect to ${redirectEndpoint} if error 403` + - ' occured', done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: endpoint, - responseType: 'redirect', - redirectUrl: `${redirectEndpoint}/`, - }, done); + it(`should redirect to ${redirectEndpoint} if error 403` + ' occured', done => { + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: endpoint, + responseType: 'redirect', + redirectUrl: `${redirectEndpoint}/`, + }, + done, + ); }); }); @@ -408,23 +471,23 @@ describe('User visits bucket website endpoint', () => { HostName: 'www.google.com', }; webConfig.addRoutingRule(redirect, condition); - s3.putBucketWebsite({ Bucket: bucket, - WebsiteConfiguration: webConfig }, done); + s3.putBucketWebsite({ Bucket: bucket, WebsiteConfiguration: webConfig }, done); }); - it(`should redirect to ${redirectEndpoint}/about/ if ` + - 'key prefix is equal to "about"', done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: `${endpoint}/about/`, - responseType: 'redirect', - redirectUrl: `${redirectEndpoint}/about/`, - }, done); + it(`should redirect to ${redirectEndpoint}/about/ if ` + 'key prefix is equal to "about"', done => { + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: `${endpoint}/about/`, + responseType: 'redirect', + redirectUrl: `${redirectEndpoint}/about/`, + }, + done, + ); }); }); - describe.skip('redirect to hostname with prefix and error condition', - () => { + describe.skip('redirect to hostname with prefix and error condition', () => { beforeEach(done => { const webConfig = new WebsiteConfigTester('index.html'); const condition = { @@ -435,19 +498,23 @@ describe('User visits bucket website endpoint', () => { HostName: 'www.google.com', }; webConfig.addRoutingRule(redirect, condition); - s3.putBucketWebsite({ Bucket: bucket, - WebsiteConfiguration: webConfig }, done); + s3.putBucketWebsite({ Bucket: bucket, WebsiteConfiguration: webConfig }, done); }); - it(`should redirect to ${redirectEndpoint} if ` + - 'key prefix is equal to "about" AND error code 403', done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: `${endpoint}/about/`, - responseType: 'redirect', - redirectUrl: `${redirectEndpoint}/about/`, - }, done); - }); + it( + `should redirect to ${redirectEndpoint} if ` + 'key prefix is equal to "about" AND error code 403', + done => { + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: `${endpoint}/about/`, + responseType: 'redirect', + redirectUrl: `${redirectEndpoint}/about/`, + }, + done, + ); + }, + ); }); describe.skip('redirect with multiple redirect rules', () => { @@ -464,22 +531,23 @@ describe('User visits bucket website endpoint', () => { }; webConfig.addRoutingRule(redirectOne, conditions); webConfig.addRoutingRule(redirectTwo, conditions); - s3.putBucketWebsite({ Bucket: bucket, - WebsiteConfiguration: webConfig }, done); + s3.putBucketWebsite({ Bucket: bucket, WebsiteConfiguration: webConfig }, done); }); it('should redirect to the first one', done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: `${endpoint}/about/`, - responseType: 'redirect', - redirectUrl: `${redirectEndpoint}/about/`, - }, done); + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: `${endpoint}/about/`, + responseType: 'redirect', + redirectUrl: `${redirectEndpoint}/about/`, + }, + done, + ); }); }); - describe.skip('redirect with protocol', - () => { + describe.skip('redirect with protocol', () => { beforeEach(done => { const webConfig = new WebsiteConfigTester('index.html'); const condition = { @@ -490,18 +558,19 @@ describe('User visits bucket website endpoint', () => { HostName: 'www.google.com', }; webConfig.addRoutingRule(redirect, condition); - s3.putBucketWebsite({ Bucket: bucket, - WebsiteConfiguration: webConfig }, done); + s3.putBucketWebsite({ Bucket: bucket, WebsiteConfiguration: webConfig }, done); }); - it('should redirect to https://www.google.com/about if ' + - 'https protocols', done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: `${endpoint}/about/`, - responseType: 'redirect', - redirectUrl: 'https://www.google.com/about/', - }, done); + it('should redirect to https://www.google.com/about if ' + 'https protocols', done => { + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: `${endpoint}/about/`, + responseType: 'redirect', + redirectUrl: 'https://www.google.com/about/', + }, + done, + ); }); }); @@ -513,23 +582,23 @@ describe('User visits bucket website endpoint', () => { const redirect = { ReplaceKeyWith: 'redirect.html', }; - putBucketWebsiteAndPutObjectRedirect(redirect, condition, - 'redirect.html', done); + putBucketWebsiteAndPutObjectRedirect(redirect, condition, 'redirect.html', done); }); afterEach(done => { - s3.deleteObject({ Bucket: bucket, Key: 'redirect.html' }, - err => done(err)); + s3.deleteObject({ Bucket: bucket, Key: 'redirect.html' }, err => done(err)); }); - it('should serve redirect file if error 403 error occured', - done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: endpoint, - responseType: 'redirect-user', - redirectUrl: `${endpoint}/redirect.html`, - }, done); + it('should serve redirect file if error 403 error occured', done => { + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: endpoint, + responseType: 'redirect-user', + redirectUrl: `${endpoint}/redirect.html`, + }, + done, + ); }); }); @@ -544,23 +613,23 @@ describe('User visits bucket website endpoint', () => { ReplaceKeyPrefixWith: 'about/', }; webConfig.addRoutingRule(redirect, condition); - s3.putBucketWebsite({ Bucket: bucket, - WebsiteConfiguration: webConfig }, done); + s3.putBucketWebsite({ Bucket: bucket, WebsiteConfiguration: webConfig }, done); }); - it(`should redirect to ${redirectEndpoint}/about/ if ` + - 'ReplaceKeyPrefixWith equals "about/"', done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: endpoint, - responseType: 'redirect', - redirectUrl: `${redirectEndpoint}/about/`, - }, done); + it(`should redirect to ${redirectEndpoint}/about/ if ` + 'ReplaceKeyPrefixWith equals "about/"', done => { + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: endpoint, + responseType: 'redirect', + redirectUrl: `${redirectEndpoint}/about/`, + }, + done, + ); }); }); - describe.skip('redirect requests with prefix /about to redirect/', - () => { + describe.skip('redirect requests with prefix /about to redirect/', () => { beforeEach(done => { const condition = { KeyPrefixEquals: 'about/', @@ -568,184 +637,209 @@ describe('User visits bucket website endpoint', () => { const redirect = { ReplaceKeyPrefixWith: 'redirect/', }; - putBucketWebsiteAndPutObjectRedirect(redirect, condition, - 'redirect/index.html', done); + putBucketWebsiteAndPutObjectRedirect(redirect, condition, 'redirect/index.html', done); }); afterEach(done => { - s3.deleteObject({ Bucket: bucket, Key: 'redirect/index.html' }, - err => done(err)); + s3.deleteObject({ Bucket: bucket, Key: 'redirect/index.html' }, err => done(err)); }); - it('should serve redirect file if key prefix is equal to "about"', - done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: `${endpoint}/about/`, - responseType: 'redirect-user', - redirectUrl: `${endpoint}/redirect/`, - }, done); + it('should serve redirect file if key prefix is equal to "about"', done => { + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: `${endpoint}/about/`, + responseType: 'redirect-user', + redirectUrl: `${endpoint}/redirect/`, + }, + done, + ); }); }); - describe.skip('redirect requests, with prefix /about and that return ' + - '403 error, to prefix redirect/', () => { - beforeEach(done => { - const condition = { - KeyPrefixEquals: 'about/', - HttpErrorCodeReturnedEquals: '403', - }; - const redirect = { - ReplaceKeyPrefixWith: 'redirect/', - }; - putBucketWebsiteAndPutObjectRedirect(redirect, condition, - 'redirect/index.html', done); - }); + describe.skip( + 'redirect requests, with prefix /about and that return ' + '403 error, to prefix redirect/', + () => { + beforeEach(done => { + const condition = { + KeyPrefixEquals: 'about/', + HttpErrorCodeReturnedEquals: '403', + }; + const redirect = { + ReplaceKeyPrefixWith: 'redirect/', + }; + putBucketWebsiteAndPutObjectRedirect(redirect, condition, 'redirect/index.html', done); + }); - afterEach(done => { - s3.deleteObject({ Bucket: bucket, Key: 'redirect/index.html' }, - err => done(err)); - }); + afterEach(done => { + s3.deleteObject({ Bucket: bucket, Key: 'redirect/index.html' }, err => done(err)); + }); - it('should serve redirect file if key prefix is equal to ' + - '"about" and error 403', - done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: `${endpoint}/about/`, - responseType: 'redirect-user', - redirectUrl: `${endpoint}/redirect/`, - }, done); - }); - }); + it('should serve redirect file if key prefix is equal to ' + '"about" and error 403', done => { + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: `${endpoint}/about/`, + responseType: 'redirect-user', + redirectUrl: `${endpoint}/redirect/`, + }, + done, + ); + }); + }, + ); describe('object redirect to /', () => { beforeEach(done => { const webConfig = new WebsiteConfigTester('index.html'); - s3.putBucketWebsite({ Bucket: bucket, - WebsiteConfiguration: webConfig }, err => { - assert.strictEqual(err, - null, `Found unexpected err ${err}`); - s3.putObject({ Bucket: bucket, Key: 'index.html', - ACL: 'public-read', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/index.html')), - ContentType: 'text/html', - Metadata: { - test: 'value', + s3.putBucketWebsite({ Bucket: bucket, WebsiteConfiguration: webConfig }, err => { + assert.strictEqual(err, null, `Found unexpected err ${err}`); + s3.putObject( + { + Bucket: bucket, + Key: 'index.html', + ACL: 'public-read', + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/index.html')), + ContentType: 'text/html', + Metadata: { + test: 'value', + }, + WebsiteRedirectLocation: '/', }, - WebsiteRedirectLocation: '/', - }, err => { assert.strictEqual(err, null); done(); - }); + }, + ); }); }); afterEach(done => { - s3.deleteObject({ Bucket: bucket, Key: 'index.html' }, - err => done(err)); + s3.deleteObject({ Bucket: bucket, Key: 'index.html' }, err => done(err)); }); it('should redirect to /', done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: `${endpoint}/index.html`, - responseType: 'redirect', - redirectUrl: '/', - }, done); + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: `${endpoint}/index.html`, + responseType: 'redirect', + redirectUrl: '/', + }, + done, + ); }); }); describe('with bucket policy', () => { beforeEach(done => { - const webConfig = new WebsiteConfigTester('index.html', - 'error.html'); - - async.waterfall([ - next => s3.putBucketWebsite({ Bucket: bucket, - WebsiteConfiguration: webConfig }, next), - (data, next) => s3.putBucketPolicy({ Bucket: bucket, - Policy: JSON.stringify({ - Version: '2012-10-17', - Statement: [{ - Sid: 'PublicReadGetObject', - Effect: 'Allow', - Principal: '*', - Action: ['s3:GetObject'], - Resource: [ - `arn:aws:s3:::${bucket}/index.html`, - `arn:aws:s3:::${bucket}/error.html`, - `arn:aws:s3:::${bucket}/access.html`, - ], - }, - { - Sid: 'DenyUnrelatedObj', - Effect: 'Deny', - Principal: '*', - Action: ['s3:GetObject'], - Resource: [ - `arn:aws:s3:::${bucket}/unrelated_obj.html`, - ], - }], - }), - }, next), - (data, next) => s3.putObject({ - Bucket: bucket, Key: 'index.html', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/index.html')), - ContentType: 'text/html', - }, next), - (data, next) => s3.putObject({ - Bucket: bucket, Key: 'error.html', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/error.html')), - ContentType: 'text/html', - }, next), - - ], err => { - assert.ifError(err); - done(); - }); + const webConfig = new WebsiteConfigTester('index.html', 'error.html'); + + async.waterfall( + [ + next => s3.putBucketWebsite({ Bucket: bucket, WebsiteConfiguration: webConfig }, next), + (data, next) => + s3.putBucketPolicy( + { + Bucket: bucket, + Policy: JSON.stringify({ + Version: '2012-10-17', + Statement: [ + { + Sid: 'PublicReadGetObject', + Effect: 'Allow', + Principal: '*', + Action: ['s3:GetObject'], + Resource: [ + `arn:aws:s3:::${bucket}/index.html`, + `arn:aws:s3:::${bucket}/error.html`, + `arn:aws:s3:::${bucket}/access.html`, + ], + }, + { + Sid: 'DenyUnrelatedObj', + Effect: 'Deny', + Principal: '*', + Action: ['s3:GetObject'], + Resource: [`arn:aws:s3:::${bucket}/unrelated_obj.html`], + }, + ], + }), + }, + next, + ), + (data, next) => + s3.putObject( + { + Bucket: bucket, + Key: 'index.html', + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/index.html')), + ContentType: 'text/html', + }, + next, + ), + (data, next) => + s3.putObject( + { + Bucket: bucket, + Key: 'error.html', + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/error.html')), + ContentType: 'text/html', + }, + next, + ), + ], + err => { + assert.ifError(err); + done(); + }, + ); }); afterEach(done => { - async.waterfall([ - next => s3.deleteObject({ Bucket: bucket, - Key: 'index.html' }, next), - (data, next) => s3.deleteObject({ Bucket: bucket, - Key: 'error.html' }, next), - ], err => { - assert.ifError(err); - done(); - }); + async.waterfall( + [ + next => s3.deleteObject({ Bucket: bucket, Key: 'index.html' }, next), + (data, next) => s3.deleteObject({ Bucket: bucket, Key: 'error.html' }, next), + ], + err => { + assert.ifError(err); + done(); + }, + ); }); it('should serve indexDocument if no key requested', done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: endpoint, - responseType: 'index-user', - }, done); + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: endpoint, + responseType: 'index-user', + }, + done, + ); }); - it('should serve custom error 403 with deny on unrelated object ' + - 'and no access to key', done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: `${endpoint}/non_existing.html`, - responseType: 'error-user', - }, done); + it('should serve custom error 403 with deny on unrelated object ' + 'and no access to key', done => { + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: `${endpoint}/non_existing.html`, + responseType: 'error-user', + }, + done, + ); }); - it('should serve custom error 404 with deny on unrelated object ' + - 'and access to key', done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: `${endpoint}/access.html`, - responseType: 'error-user-404', - }, done); + it('should serve custom error 404 with deny on unrelated object ' + 'and access to key', done => { + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: `${endpoint}/access.html`, + responseType: 'error-user-404', + }, + done, + ); }); }); @@ -759,34 +853,37 @@ describe('User visits bucket website endpoint', () => { ReplaceKeyWith: 'whatever.html', }; webConfig.addRoutingRule(redirect, condition); - s3.putBucketWebsite({ Bucket: bucket, - WebsiteConfiguration: webConfig }, err => { - assert.strictEqual(err, - null, `Found unexpected err ${err}`); - s3.putObject({ Bucket: bucket, Key: 'index.html', - ACL: 'public-read', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/index.html')), - ContentType: 'text/html', - }, + s3.putBucketWebsite({ Bucket: bucket, WebsiteConfiguration: webConfig }, err => { + assert.strictEqual(err, null, `Found unexpected err ${err}`); + s3.putObject( + { + Bucket: bucket, + Key: 'index.html', + ACL: 'public-read', + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/index.html')), + ContentType: 'text/html', + }, err => { assert.strictEqual(err, null); done(); - }); + }, + ); }); }); afterEach(done => { - s3.deleteObject({ Bucket: bucket, Key: 'index.html' }, - err => done(err)); + s3.deleteObject({ Bucket: bucket, Key: 'index.html' }, err => done(err)); }); it('should not redirect if index key is not explicit', done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: endpoint, - responseType: 'index-user', - }, done); + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: endpoint, + responseType: 'index-user', + }, + done, + ); }); }); @@ -795,72 +892,84 @@ describe('User visits bucket website endpoint', () => { const webConfig = new WebsiteConfigTester('index.html'); const object = { Bucket: bucket, - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/index.html')), + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/index.html')), ContentType: 'text/html', }; - async.waterfall([ - next => s3.putBucketWebsite({ Bucket: bucket, - WebsiteConfiguration: webConfig }, next), - (data, next) => s3.putBucketPolicy({ Bucket: bucket, - Policy: JSON.stringify({ - Version: '2012-10-17', - Statement: [{ - Sid: 'PublicReadGetObject', - Effect: 'Allow', - Principal: '*', - Action: ['s3:GetObject'], - Resource: [ - `arn:aws:s3:::${bucket}/original_key_file`, - `arn:aws:s3:::${bucket}/original_key_nofile`, - `arn:aws:s3:::${bucket}/file/*`, - `arn:aws:s3:::${bucket}/nofile/*`, - ], - }], - }), - }, next), - (data, next) => s3.putObject(Object.assign({}, object, - { Key: 'original_key_file/index.html' }), next), - (data, next) => s3.putObject(Object.assign({}, object, - { Key: 'file/index.html' }), next), // the redirect 302 - (data, next) => s3.putObject(Object.assign({}, object, - { Key: 'no_access_file/index.html' }), next), - ], err => { - assert.ifError(err); - done(); - }); + async.waterfall( + [ + next => s3.putBucketWebsite({ Bucket: bucket, WebsiteConfiguration: webConfig }, next), + (data, next) => + s3.putBucketPolicy( + { + Bucket: bucket, + Policy: JSON.stringify({ + Version: '2012-10-17', + Statement: [ + { + Sid: 'PublicReadGetObject', + Effect: 'Allow', + Principal: '*', + Action: ['s3:GetObject'], + Resource: [ + `arn:aws:s3:::${bucket}/original_key_file`, + `arn:aws:s3:::${bucket}/original_key_nofile`, + `arn:aws:s3:::${bucket}/file/*`, + `arn:aws:s3:::${bucket}/nofile/*`, + ], + }, + ], + }), + }, + next, + ), + (data, next) => + s3.putObject(Object.assign({}, object, { Key: 'original_key_file/index.html' }), next), + (data, next) => s3.putObject(Object.assign({}, object, { Key: 'file/index.html' }), next), // the redirect 302 + (data, next) => + s3.putObject(Object.assign({}, object, { Key: 'no_access_file/index.html' }), next), + ], + err => { + assert.ifError(err); + done(); + }, + ); }); afterEach(done => { - async.waterfall([ - next => s3.deleteObject({ Bucket: bucket, - Key: 'original_key_file/index.html' }, next), - (data, next) => s3.deleteObject({ Bucket: bucket, - Key: 'file/index.html' }, next), - (data, next) => s3.deleteObject({ Bucket: bucket, - Key: 'no_access_file/index.html' }, next), - ], err => { - assert.ifError(err); - done(); - }); + async.waterfall( + [ + next => s3.deleteObject({ Bucket: bucket, Key: 'original_key_file/index.html' }, next), + (data, next) => s3.deleteObject({ Bucket: bucket, Key: 'file/index.html' }, next), + (data, next) => s3.deleteObject({ Bucket: bucket, Key: 'no_access_file/index.html' }, next), + ], + err => { + assert.ifError(err); + done(); + }, + ); }); it('should redirect 302 with trailing / on folder with index', done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: `${endpoint}/file`, - responseType: 'redirect-error-found', - redirectUrl: '/file/', - }, done); + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: `${endpoint}/file`, + responseType: 'redirect-error-found', + redirectUrl: '/file/', + }, + done, + ); }); - it('should return 404 on original key access without index', - done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: `${endpoint}/original_key_nofile`, - responseType: '404-not-found', - }, done); + it('should return 404 on original key access without index', done => { + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: `${endpoint}/original_key_nofile`, + responseType: '404-not-found', + }, + done, + ); }); describe('should return 403', () => { @@ -879,12 +988,16 @@ describe('User visits bucket website endpoint', () => { }, ].forEach(test => it(test.it, done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: `${endpoint}/${test.key}`, - responseType: '403-access-denied', - }, done); - })); + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: `${endpoint}/${test.key}`, + responseType: '403-access-denied', + }, + done, + ); + }), + ); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/object/websiteGetWithACL.js b/tests/functional/aws-node-sdk/test/object/websiteGetWithACL.js index 7bc84b7fd2..cb63aaafc6 100644 --- a/tests/functional/aws-node-sdk/test/object/websiteGetWithACL.js +++ b/tests/functional/aws-node-sdk/test/object/websiteGetWithACL.js @@ -12,13 +12,11 @@ const s3 = new S3Client(config); // `127.0.0.1 bucketwebsitetester.s3-website-us-east-1.amazonaws.com` const transport = conf.https ? 'https' : 'http'; -const bucket = process.env.AWS_ON_AIR ? 'awsbucketwebsitetester' : - 'bucketwebsitetester'; -const hostname = process.env.S3_END_TO_END ? - `${bucket}.s3-website-us-east-1.scality.com` : - `${bucket}.s3-website-us-east-1.amazonaws.com`; -const endpoint = process.env.AWS_ON_AIR ? `${transport}://${hostname}` : - `${transport}://${hostname}:8000`; +const bucket = process.env.AWS_ON_AIR ? 'awsbucketwebsitetester' : 'bucketwebsitetester'; +const hostname = process.env.S3_END_TO_END + ? `${bucket}.s3-website-us-east-1.scality.com` + : `${bucket}.s3-website-us-east-1.amazonaws.com`; +const endpoint = process.env.AWS_ON_AIR ? `${transport}://${hostname}` : `${transport}://${hostname}:8000`; const aclEquivalent = { public: ['public-read-write', 'public-read'], @@ -33,87 +31,75 @@ const aclTests = [ html: '403-access-denied', }, { - it: 'should return 403 if public bucket - private index - public ' + - 'error documents', + it: 'should return 403 if public bucket - private index - public ' + 'error documents', bucketACL: 'public', objects: { index: 'private', error: 'private' }, html: '403-access-denied', }, { - it: 'should return index doc if private bucket - public index - ' + - 'public error documents', + it: 'should return index doc if private bucket - public index - ' + 'public error documents', bucketACL: 'private', objects: { index: 'public-read', error: 'private' }, html: 'index-user', }, { - it: 'should return index doc if public bucket - public index - ' + - 'private error documents', + it: 'should return index doc if public bucket - public index - ' + 'private error documents', bucketACL: 'public', objects: { index: 'public-read', error: 'private' }, html: 'index-user', }, { - it: 'should return index doc if private bucket - public index - ' + - 'public error documents', + it: 'should return index doc if private bucket - public index - ' + 'public error documents', bucketACL: 'private', objects: { index: 'public-read', error: 'public-read' }, html: 'index-user', }, { - it: 'should return index doc if public bucket - public index - ' + - 'public error documents', + it: 'should return index doc if public bucket - public index - ' + 'public error documents', bucketACL: 'public', objects: { index: 'public-read', error: 'public-read' }, html: 'index-user', }, { - it: 'should return error doc if private bucket - without index - ' + - 'public error documents', + it: 'should return error doc if private bucket - without index - ' + 'public error documents', bucketACL: 'private', objects: { error: 'public-read' }, html: 'error-user', }, { - it: 'should return 404 if public bucket - without index - ' + - 'public error documents', + it: 'should return 404 if public bucket - without index - ' + 'public error documents', bucketACL: 'public', objects: { error: 'public-read' }, html: 'error-user-404', }, { - it: 'should return 403 if private bucket - without index - ' + - 'private error documents', + it: 'should return 403 if private bucket - without index - ' + 'private error documents', bucketACL: 'private', objects: { error: 'private' }, html: '403-access-denied', }, { - it: 'should return 404 if public bucket - without index - ' + - 'private error documents', + it: 'should return 404 if public bucket - without index - ' + 'private error documents', bucketACL: 'public', objects: { error: 'private' }, html: '404-not-found', }, { - it: 'should return 404 if public bucket - without index - ' + - 'without error documents', + it: 'should return 404 if public bucket - without index - ' + 'without error documents', bucketACL: 'public', - objects: { }, + objects: {}, html: '404-not-found', }, { - it: 'should return 403 if private bucket - without index - ' + - 'without error documents', + it: 'should return 403 if private bucket - without index - ' + 'without error documents', bucketACL: 'private', - objects: { }, + objects: {}, html: '403-access-denied', }, - ]; describe('User visits bucket website endpoint with ACL', () => { @@ -121,12 +107,10 @@ describe('User visits bucket website endpoint with ACL', () => { aclEquivalent[test.bucketACL].forEach(bucketACL => { describe(`with existing bucket with ${bucketACL} acl`, () => { beforeEach(done => { - WebsiteConfigTester.createPutBucketWebsite(s3, bucket, - bucketACL, test.objects, done); + WebsiteConfigTester.createPutBucketWebsite(s3, bucket, bucketACL, test.objects, done); }); afterEach(done => { - WebsiteConfigTester.deleteObjectsThenBucket(s3, bucket, - test.objects, err => { + WebsiteConfigTester.deleteObjectsThenBucket(s3, bucket, test.objects, err => { if (process.env.AWS_ON_AIR) { // Give some time for AWS to finish deleting // object and buckets before starting next test @@ -138,29 +122,38 @@ describe('User visits bucket website endpoint with ACL', () => { }); it(`${test.it} with no auth credentials sent`, done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: endpoint, - requestType: test.html, - }, done); + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: endpoint, + requestType: test.html, + }, + done, + ); }); it(`${test.it} even with invalid auth credentials`, done => { - WebsiteConfigTester.checkHTML({ - auth: 'invalid credentials', - method: 'GET', - url: endpoint, - requestType: test.html, - }, done); + WebsiteConfigTester.checkHTML( + { + auth: 'invalid credentials', + method: 'GET', + url: endpoint, + requestType: test.html, + }, + done, + ); }); it(`${test.it} even with valid auth credentials`, done => { - WebsiteConfigTester.checkHTML({ - auth: 'valid credentials', - method: 'GET', - url: endpoint, - requestType: test.html, - }, done); + WebsiteConfigTester.checkHTML( + { + auth: 'valid credentials', + method: 'GET', + url: endpoint, + requestType: test.html, + }, + done, + ); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/object/websiteHead.js b/tests/functional/aws-node-sdk/test/object/websiteHead.js index 0d26fda09a..bbdccf4cde 100644 --- a/tests/functional/aws-node-sdk/test/object/websiteHead.js +++ b/tests/functional/aws-node-sdk/test/object/websiteHead.js @@ -23,19 +23,16 @@ const s3 = new S3Client(config); // `127.0.0.1 bucketwebsitetester.s3-website-us-east-1.amazonaws.com` const transport = conf.https ? 'https' : 'http'; -const bucket = process.env.AWS_ON_AIR ? `awsbucketwebsitetester-${Date.now()}` : - 'bucketwebsitetester'; -const hostname = process.env.S3_END_TO_END ? - `${bucket}.s3-website-us-east-1.scality.com` : - `${bucket}.s3-website-us-east-1.amazonaws.com`; -const endpoint = process.env.AWS_ON_AIR ? `${transport}://${hostname}` : - `${transport}://${hostname}:8000`; -const redirectEndpoint = conf.https ? 'https://www.google.com/' : - 'http://www.google.com/'; +const bucket = process.env.AWS_ON_AIR ? `awsbucketwebsitetester-${Date.now()}` : 'bucketwebsitetester'; +const hostname = process.env.S3_END_TO_END + ? `${bucket}.s3-website-us-east-1.scality.com` + : `${bucket}.s3-website-us-east-1.amazonaws.com`; +const endpoint = process.env.AWS_ON_AIR ? `${transport}://${hostname}` : `${transport}://${hostname}:8000`; +const redirectEndpoint = conf.https ? 'https://www.google.com/' : 'http://www.google.com/'; const indexDocETag = '"95a589c37a2df74b062fb4d5a6f64197"'; const indexExpectedHeaders = { - 'etag': indexDocETag, + etag: indexDocETag, 'x-amz-meta-test': 'value', }; @@ -82,7 +79,6 @@ const indexExpectedHeaders = { // KX/MgqE4dZCJ4d9eF59Wbg/kza40cWcoA= // x-amz-request-id: 0073330F58C7137C - describe('Head request on bucket website endpoint', () => { it('should return 404 when no such bucket', done => { const expectedHeaders = { @@ -91,8 +87,7 @@ describe('Head request on bucket website endpoint', () => { // so compatible with aws 'x-amz-error-message': 'The specified bucket does not exist.', }; - WebsiteConfigTester.makeHeadRequest(undefined, endpoint, 404, - expectedHeaders, done); + WebsiteConfigTester.makeHeadRequest(undefined, endpoint, 404, expectedHeaders, done); }); describe('with existing bucket', () => { @@ -103,95 +98,116 @@ describe('Head request on bucket website endpoint', () => { it('should return 404 when no website configuration', done => { const expectedHeaders = { 'x-amz-error-code': 'NoSuchWebsiteConfiguration', - 'x-amz-error-message': 'The specified bucket does not ' + - 'have a website configuration', + 'x-amz-error-message': 'The specified bucket does not ' + 'have a website configuration', }; - WebsiteConfigTester.makeHeadRequest(undefined, endpoint, 404, - expectedHeaders, done); + WebsiteConfigTester.makeHeadRequest(undefined, endpoint, 404, expectedHeaders, done); }); describe('with existing configuration', () => { beforeEach(async () => { const webConfig = new WebsiteConfigTester('index.html'); - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })); - await s3.send(new PutObjectCommand({ Bucket: bucket, Key: 'index.html', - ACL: 'public-read', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/index.html')), + await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: 'index.html', + ACL: 'public-read', + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/index.html')), ContentType: 'text/html', Metadata: { test: 'value', }, - })); + }), + ); }); afterEach(() => s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: 'index.html' }))); - it('should return indexDocument headers if no key ' + - 'requested', done => { - WebsiteConfigTester.makeHeadRequest(undefined, endpoint, - 200, indexExpectedHeaders, done); + it('should return indexDocument headers if no key ' + 'requested', done => { + WebsiteConfigTester.makeHeadRequest(undefined, endpoint, 200, indexExpectedHeaders, done); }); it('should return indexDocument headers if key requested', done => { - WebsiteConfigTester.makeHeadRequest(undefined, - `${endpoint}/index.html`, 200, indexExpectedHeaders, done); + WebsiteConfigTester.makeHeadRequest( + undefined, + `${endpoint}/index.html`, + 200, + indexExpectedHeaders, + done, + ); }); }); describe('with path prefix in request with/without key', () => { beforeEach(async () => { const webConfig = new WebsiteConfigTester('index.html'); - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })).catch(err => { - assert.strictEqual(err, null, `Found unexpected err ${err}`); - }); - await s3.send(new PutObjectCommand({ Bucket: bucket, - Key: 'pathprefix/index.html', - ACL: 'public-read', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/index.html')), - ContentType: 'text/html', - Metadata: { - test: 'value', - }, - })).catch(err => { + await s3 + .send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })) + .catch(err => { + assert.strictEqual(err, null, `Found unexpected err ${err}`); + }); + await s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: 'pathprefix/index.html', + ACL: 'public-read', + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/index.html')), + ContentType: 'text/html', + Metadata: { + test: 'value', + }, + }), + ) + .catch(err => { assert.strictEqual(err, null); }); }); afterEach(async () => s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: 'pathprefix/index.html' }))); - it('should serve indexDocument if path request without key', - done => { - WebsiteConfigTester.makeHeadRequest(undefined, - `${endpoint}/pathprefix/`, 200, indexExpectedHeaders, done); - }); - - it('should serve indexDocument if path request with key', - done => { - WebsiteConfigTester.makeHeadRequest(undefined, - `${endpoint}/pathprefix/index.html`, 200, - indexExpectedHeaders, done); + it('should serve indexDocument if path request without key', done => { + WebsiteConfigTester.makeHeadRequest( + undefined, + `${endpoint}/pathprefix/`, + 200, + indexExpectedHeaders, + done, + ); + }); + + it('should serve indexDocument if path request with key', done => { + WebsiteConfigTester.makeHeadRequest( + undefined, + `${endpoint}/pathprefix/index.html`, + 200, + indexExpectedHeaders, + done, + ); }); }); describe('with private key', () => { beforeEach(async () => { const webConfig = new WebsiteConfigTester('index.html'); - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })).catch(err => { - assert.strictEqual(err, null, `Found unexpected err ${err}`); - }); - await s3.send(new PutObjectCommand({ Bucket: bucket, - Key: 'index.html', - ACL: 'private', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/index.html')), - ContentType: 'text/html' })).catch(err => { - assert.strictEqual(err, null, `Found unexpected err ${err}`); - }); + await s3 + .send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })) + .catch(err => { + assert.strictEqual(err, null, `Found unexpected err ${err}`); + }); + await s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: 'index.html', + ACL: 'private', + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/index.html')), + ContentType: 'text/html', + }), + ) + .catch(err => { + assert.strictEqual(err, null, `Found unexpected err ${err}`); + }); }); afterEach(() => s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: 'index.html' }))); @@ -201,16 +217,14 @@ describe('Head request on bucket website endpoint', () => { 'x-amz-error-code': 'AccessDenied', 'x-amz-error-message': 'Access Denied', }; - WebsiteConfigTester.makeHeadRequest(undefined, endpoint, 403, - expectedHeaders, done); + WebsiteConfigTester.makeHeadRequest(undefined, endpoint, 403, expectedHeaders, done); }); }); describe('with nonexisting index document key', () => { beforeEach(async () => { const webConfig = new WebsiteConfigTester('index.html'); - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })); + await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })); }); it('should return 403 if nonexisting index document key', done => { @@ -218,8 +232,7 @@ describe('Head request on bucket website endpoint', () => { 'x-amz-error-code': 'AccessDenied', 'x-amz-error-message': 'Access Denied', }; - WebsiteConfigTester.makeHeadRequest(undefined, endpoint, 403, - expectedHeaders, done); + WebsiteConfigTester.makeHeadRequest(undefined, endpoint, 403, expectedHeaders, done); }); }); @@ -228,94 +241,95 @@ describe('Head request on bucket website endpoint', () => { const redirectAllTo = { HostName: 'www.google.com', }; - const webConfig = new WebsiteConfigTester(null, null, - redirectAllTo); - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })).catch(err => { - assert.strictEqual(err, null, `Found unexpected err ${err}`); - }); + const webConfig = new WebsiteConfigTester(null, null, redirectAllTo); + await s3 + .send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })) + .catch(err => { + assert.strictEqual(err, null, `Found unexpected err ${err}`); + }); }); it(`should redirect to ${redirectEndpoint}`, done => { const expectedHeaders = { location: redirectEndpoint, }; - WebsiteConfigTester.makeHeadRequest(undefined, - endpoint, 301, expectedHeaders, done); + WebsiteConfigTester.makeHeadRequest(undefined, endpoint, 301, expectedHeaders, done); }); it(`should redirect to ${redirectEndpoint}about`, done => { const expectedHeaders = { location: `${redirectEndpoint}about/`, }; - WebsiteConfigTester.makeHeadRequest(undefined, - `${endpoint}/about/`, 301, expectedHeaders, done); + WebsiteConfigTester.makeHeadRequest(undefined, `${endpoint}/about/`, 301, expectedHeaders, done); }); }); - describe('redirect all requests to https://www.google.com ' + - 'since https protocol set in website config', () => { - // Note: these tests will all redirect to https even if - // conf does not have https since protocol in website config - // specifies https - beforeEach(async () => { - const redirectAllTo = { - HostName: 'www.google.com', - Protocol: 'https', - }; - const webConfig = new WebsiteConfigTester(null, null, - redirectAllTo); - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })).catch(err => { - assert.strictEqual(err, null, `Found unexpected err ${err}`); + describe( + 'redirect all requests to https://www.google.com ' + 'since https protocol set in website config', + () => { + // Note: these tests will all redirect to https even if + // conf does not have https since protocol in website config + // specifies https + beforeEach(async () => { + const redirectAllTo = { + HostName: 'www.google.com', + Protocol: 'https', + }; + const webConfig = new WebsiteConfigTester(null, null, redirectAllTo); + await s3 + .send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })) + .catch(err => { + assert.strictEqual(err, null, `Found unexpected err ${err}`); + }); }); - }); - it('should redirect to https://google.com', done => { - const expectedHeaders = { - location: 'https://www.google.com/', - }; - WebsiteConfigTester.makeHeadRequest(undefined, endpoint, - 301, expectedHeaders, done); - }); + it('should redirect to https://google.com', done => { + const expectedHeaders = { + location: 'https://www.google.com/', + }; + WebsiteConfigTester.makeHeadRequest(undefined, endpoint, 301, expectedHeaders, done); + }); - it('should redirect to https://google.com/about', done => { - const expectedHeaders = { - location: 'https://www.google.com/about/', - }; - WebsiteConfigTester.makeHeadRequest(undefined, - `${endpoint}/about/`, 301, expectedHeaders, done); - }); - }); + it('should redirect to https://google.com/about', done => { + const expectedHeaders = { + location: 'https://www.google.com/about/', + }; + WebsiteConfigTester.makeHeadRequest(undefined, `${endpoint}/about/`, 301, expectedHeaders, done); + }); + }, + ); describe('with custom error document', () => { beforeEach(async () => { - const webConfig = new WebsiteConfigTester('index.html', - 'error.html'); - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })).catch(err => { - assert.strictEqual(err, null, `Found unexpected err ${err}`); - }); - await s3.send(new PutObjectCommand({ Bucket: bucket, - Key: 'error.html', - ACL: 'public-read', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/error.html')), - ContentType: 'text/html' })).catch(err => { - assert.strictEqual(err, null, `Found unexpected err ${err}`); - }); + const webConfig = new WebsiteConfigTester('index.html', 'error.html'); + await s3 + .send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })) + .catch(err => { + assert.strictEqual(err, null, `Found unexpected err ${err}`); + }); + await s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: 'error.html', + ACL: 'public-read', + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/error.html')), + ContentType: 'text/html', + }), + ) + .catch(err => { + assert.strictEqual(err, null, `Found unexpected err ${err}`); + }); }); afterEach(() => s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: 'error.html' }))); - it('should return regular error headers regardless of whether ' + - 'custom error document', done => { + it('should return regular error headers regardless of whether ' + 'custom error document', done => { const expectedHeaders = { 'x-amz-error-code': 'AccessDenied', 'x-amz-error-message': 'Access Denied', }; - WebsiteConfigTester.makeHeadRequest(undefined, - `${endpoint}/madeup`, 403, expectedHeaders, done); + WebsiteConfigTester.makeHeadRequest(undefined, `${endpoint}/madeup`, 403, expectedHeaders, done); }); }); @@ -329,19 +343,18 @@ describe('Head request on bucket website endpoint', () => { HostName: 'www.google.com', }; webConfig.addRoutingRule(redirect, condition); - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })).catch(err => { - assert.strictEqual(err, null, `Found unexpected err ${err}`); - }); + await s3 + .send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })) + .catch(err => { + assert.strictEqual(err, null, `Found unexpected err ${err}`); + }); }); - it(`should redirect to ${redirectEndpoint} if error 403` + - ' occured', done => { + it(`should redirect to ${redirectEndpoint} if error 403` + ' occured', done => { const expectedHeaders = { location: redirectEndpoint, }; - WebsiteConfigTester.makeHeadRequest(undefined, endpoint, 301, - expectedHeaders, done); + WebsiteConfigTester.makeHeadRequest(undefined, endpoint, 301, expectedHeaders, done); }); }); @@ -355,24 +368,22 @@ describe('Head request on bucket website endpoint', () => { HostName: 'www.google.com', }; webConfig.addRoutingRule(redirect, condition); - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })).catch(err => { - assert.strictEqual(err, null, `Found unexpected err ${err}`); - }); + await s3 + .send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })) + .catch(err => { + assert.strictEqual(err, null, `Found unexpected err ${err}`); + }); }); - it(`should redirect to ${redirectEndpoint}about if ` + - 'key prefix is equal to "about"', done => { + it(`should redirect to ${redirectEndpoint}about if ` + 'key prefix is equal to "about"', done => { const expectedHeaders = { location: `${redirectEndpoint}about/`, }; - WebsiteConfigTester.makeHeadRequest(undefined, - `${endpoint}/about/`, 301, expectedHeaders, done); + WebsiteConfigTester.makeHeadRequest(undefined, `${endpoint}/about/`, 301, expectedHeaders, done); }); }); - describe('redirect to hostname with prefix and error condition', - () => { + describe('redirect to hostname with prefix and error condition', () => { beforeEach(async () => { const webConfig = new WebsiteConfigTester('index.html'); const condition = { @@ -383,20 +394,22 @@ describe('Head request on bucket website endpoint', () => { HostName: 'www.google.com', }; webConfig.addRoutingRule(redirect, condition); - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })).catch(err => { - assert.strictEqual(err, null, `Found unexpected err ${err}`); - }); + await s3 + .send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })) + .catch(err => { + assert.strictEqual(err, null, `Found unexpected err ${err}`); + }); }); - it(`should redirect to ${redirectEndpoint} if ` + - 'key prefix is equal to "about" AND error code 403', done => { - const expectedHeaders = { - location: `${redirectEndpoint}about/`, - }; - WebsiteConfigTester.makeHeadRequest(undefined, - `${endpoint}/about/`, 301, expectedHeaders, done); - }); + it( + `should redirect to ${redirectEndpoint} if ` + 'key prefix is equal to "about" AND error code 403', + done => { + const expectedHeaders = { + location: `${redirectEndpoint}about/`, + }; + WebsiteConfigTester.makeHeadRequest(undefined, `${endpoint}/about/`, 301, expectedHeaders, done); + }, + ); }); describe('redirect with multiple redirect rules', () => { @@ -413,23 +426,22 @@ describe('Head request on bucket website endpoint', () => { }; webConfig.addRoutingRule(redirectOne, conditions); webConfig.addRoutingRule(redirectTwo, conditions); - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })).catch(err => { - assert.strictEqual(err, null, `Found unexpected err ${err}`); - }); + await s3 + .send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })) + .catch(err => { + assert.strictEqual(err, null, `Found unexpected err ${err}`); + }); }); it('should redirect based on first rule', done => { const expectedHeaders = { location: `${redirectEndpoint}about/`, }; - WebsiteConfigTester.makeHeadRequest(undefined, - `${endpoint}/about/`, 301, expectedHeaders, done); + WebsiteConfigTester.makeHeadRequest(undefined, `${endpoint}/about/`, 301, expectedHeaders, done); }); }); - describe('redirect with protocol', - () => { + describe('redirect with protocol', () => { beforeEach(async () => { const webConfig = new WebsiteConfigTester('index.html'); const condition = { @@ -440,19 +452,18 @@ describe('Head request on bucket website endpoint', () => { HostName: 'www.google.com', }; webConfig.addRoutingRule(redirect, condition); - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })).catch(err => { - assert.strictEqual(err, null, `Found unexpected err ${err}`); - }); + await s3 + .send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })) + .catch(err => { + assert.strictEqual(err, null, `Found unexpected err ${err}`); + }); }); - it('should redirect to https://www.google.com/about if ' + - 'https protocol specified', done => { + it('should redirect to https://www.google.com/about if ' + 'https protocol specified', done => { const expectedHeaders = { location: 'https://www.google.com/about/', }; - WebsiteConfigTester.makeHeadRequest(undefined, - `${endpoint}/about/`, 301, expectedHeaders, done); + WebsiteConfigTester.makeHeadRequest(undefined, `${endpoint}/about/`, 301, expectedHeaders, done); }); }); @@ -466,21 +477,20 @@ describe('Head request on bucket website endpoint', () => { ReplaceKeyWith: 'redirect.html', }; webConfig.addRoutingRule(redirect, condition); - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })).catch(err => { - assert.strictEqual(err, null, `Found unexpected err ${err}`); - }); + await s3 + .send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })) + .catch(err => { + assert.strictEqual(err, null, `Found unexpected err ${err}`); + }); }); afterEach(() => s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: 'redirect.html' }))); - it('should redirect to specified file if 403 error ' + - 'error occured', done => { + it('should redirect to specified file if 403 error ' + 'error occured', done => { const expectedHeaders = { location: `${endpoint}/redirect.html`, }; - WebsiteConfigTester.makeHeadRequest(undefined, endpoint, 301, - expectedHeaders, done); + WebsiteConfigTester.makeHeadRequest(undefined, endpoint, 301, expectedHeaders, done); }); }); @@ -495,24 +505,22 @@ describe('Head request on bucket website endpoint', () => { ReplaceKeyPrefixWith: 'about', }; webConfig.addRoutingRule(redirect, condition); - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })).catch(err => { - assert.strictEqual(err, null, `Found unexpected err ${err}`); - }); + await s3 + .send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })) + .catch(err => { + assert.strictEqual(err, null, `Found unexpected err ${err}`); + }); }); - it(`should redirect to ${redirectEndpoint}about if ` + - 'ReplaceKeyPrefixWith equals "about"', done => { + it(`should redirect to ${redirectEndpoint}about if ` + 'ReplaceKeyPrefixWith equals "about"', done => { const expectedHeaders = { location: `${redirectEndpoint}about`, }; - WebsiteConfigTester.makeHeadRequest(undefined, endpoint, 301, - expectedHeaders, done); + WebsiteConfigTester.makeHeadRequest(undefined, endpoint, 301, expectedHeaders, done); }); }); - describe('redirect requests with prefix /about to redirect/', - () => { + describe('redirect requests with prefix /about to redirect/', () => { beforeEach(async () => { const webConfig = new WebsiteConfigTester('index.html'); const condition = { @@ -522,27 +530,24 @@ describe('Head request on bucket website endpoint', () => { ReplaceKeyPrefixWith: 'redirect/', }; webConfig.addRoutingRule(redirect, condition); - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })).catch(err => { - assert.strictEqual(err, null, `Found unexpected err ${err}`); - }); + await s3 + .send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })) + .catch(err => { + assert.strictEqual(err, null, `Found unexpected err ${err}`); + }); }); afterEach(async () => s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: 'redirect/index.html' }))); - - it('should redirect to "redirect/" object if key prefix is equal ' + - 'to "about/"', done => { + it('should redirect to "redirect/" object if key prefix is equal ' + 'to "about/"', done => { const expectedHeaders = { location: `${endpoint}/redirect/`, }; - WebsiteConfigTester.makeHeadRequest(undefined, - `${endpoint}/about/`, 301, expectedHeaders, done); + WebsiteConfigTester.makeHeadRequest(undefined, `${endpoint}/about/`, 301, expectedHeaders, done); }); }); - describe('redirect requests, with both prefix and error code ' + - 'condition', () => { + describe('redirect requests, with both prefix and error code ' + 'condition', () => { beforeEach(async () => { const webConfig = new WebsiteConfigTester('index.html'); const condition = { @@ -553,41 +558,45 @@ describe('Head request on bucket website endpoint', () => { ReplaceKeyPrefixWith: 'redirect/', }; webConfig.addRoutingRule(redirect, condition); - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })).catch(err => { - assert.strictEqual(err, null, `Found unexpected err ${err}`); - }); + await s3 + .send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })) + .catch(err => { + assert.strictEqual(err, null, `Found unexpected err ${err}`); + }); }); afterEach(() => s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: 'redirect/index.html' }))); - it('should redirect to "redirect" object if key prefix is equal ' + - 'to "about/" and there is a 403 error satisfying the ' + - 'condition in the redirect rule', - done => { - const expectedHeaders = { - location: `${endpoint}/redirect/`, - }; - WebsiteConfigTester.makeHeadRequest(undefined, - `${endpoint}/about/`, 301, expectedHeaders, done); - }); + it( + 'should redirect to "redirect" object if key prefix is equal ' + + 'to "about/" and there is a 403 error satisfying the ' + + 'condition in the redirect rule', + done => { + const expectedHeaders = { + location: `${endpoint}/redirect/`, + }; + WebsiteConfigTester.makeHeadRequest(undefined, `${endpoint}/about/`, 301, expectedHeaders, done); + }, + ); }); describe('object redirect to /', () => { beforeEach(async () => { const webConfig = new WebsiteConfigTester('index.html'); - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })); - await s3.send(new PutObjectCommand({ Bucket: bucket, Key: 'index.html', + await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: 'index.html', ACL: 'public-read', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/index.html')), + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/index.html')), ContentType: 'text/html', Metadata: { test: 'value', }, WebsiteRedirectLocation: '/', - })); + }), + ); }); afterEach(() => s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: 'index.html' }))); @@ -596,46 +605,51 @@ describe('Head request on bucket website endpoint', () => { const expectedHeaders = { location: '/', }; - WebsiteConfigTester.makeHeadRequest(undefined, - `${endpoint}/index.html`, 301, expectedHeaders, done); + WebsiteConfigTester.makeHeadRequest(undefined, `${endpoint}/index.html`, 301, expectedHeaders, done); }); }); describe('with bucket policy', () => { beforeEach(async () => { const webConfig = new WebsiteConfigTester('index.html'); - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })); - await s3.send(new PutBucketPolicyCommand({ Bucket: bucket, Policy: JSON.stringify( - { - Version: '2012-10-17', - Statement: [{ - Sid: 'PublicReadGetObject', - Effect: 'Allow', - Principal: '*', - Action: ['s3:GetObject'], - Resource: [ - `arn:aws:s3:::${bucket}/index.html`, - `arn:aws:s3:::${bucket}/access.html`, + await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })); + await s3.send( + new PutBucketPolicyCommand({ + Bucket: bucket, + Policy: JSON.stringify({ + Version: '2012-10-17', + Statement: [ + { + Sid: 'PublicReadGetObject', + Effect: 'Allow', + Principal: '*', + Action: ['s3:GetObject'], + Resource: [ + `arn:aws:s3:::${bucket}/index.html`, + `arn:aws:s3:::${bucket}/access.html`, + ], + }, ], - }], - } - )})); - await s3.send(new PutObjectCommand({ Bucket: bucket, Key: 'index.html', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/index.html')), + }), + }), + ); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: 'index.html', + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/index.html')), ContentType: 'text/html', Metadata: { test: 'value', - }})); + }, + }), + ); }); afterEach(() => s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: 'index.html' }))); - it('should return indexDocument headers if no key ' + - 'requested', done => { - WebsiteConfigTester.makeHeadRequest(undefined, endpoint, - 200, indexExpectedHeaders, done); + it('should return indexDocument headers if no key ' + 'requested', done => { + WebsiteConfigTester.makeHeadRequest(undefined, endpoint, 200, indexExpectedHeaders, done); }); it('should serve error 403 with no access to key', done => { @@ -643,9 +657,13 @@ describe('Head request on bucket website endpoint', () => { 'x-amz-error-code': 'AccessDenied', 'x-amz-error-message': 'Access Denied', }; - WebsiteConfigTester.makeHeadRequest(undefined, - `${endpoint}/non_existing.html`, 403, expectedHeaders, - done); + WebsiteConfigTester.makeHeadRequest( + undefined, + `${endpoint}/non_existing.html`, + 403, + expectedHeaders, + done, + ); }); it('should serve error 404 with access to key', done => { @@ -653,9 +671,7 @@ describe('Head request on bucket website endpoint', () => { 'x-amz-error-code': 'NoSuchKey', 'x-amz-error-message': 'The specified key does not exist.', }; - WebsiteConfigTester.makeHeadRequest(undefined, - `${endpoint}/access.html`, 404, expectedHeaders, - done); + WebsiteConfigTester.makeHeadRequest(undefined, `${endpoint}/access.html`, 404, expectedHeaders, done); }); }); @@ -669,24 +685,25 @@ describe('Head request on bucket website endpoint', () => { ReplaceKeyWith: 'whatever.html', }; webConfig.addRoutingRule(redirect, condition); - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })); - await s3.send(new PutObjectCommand({ Bucket: bucket, Key: 'index.html', - ACL: 'public-read', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/index.html')), - ContentType: 'text/html', - Metadata: { - test: 'value', - }, - })); + await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: 'index.html', + ACL: 'public-read', + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/index.html')), + ContentType: 'text/html', + Metadata: { + test: 'value', + }, + }), + ); }); afterEach(() => s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: 'index.html' }))); it('should not redirect if index key is not explicit', done => { - WebsiteConfigTester.makeHeadRequest(undefined, endpoint, - 200, indexExpectedHeaders, done); + WebsiteConfigTester.makeHeadRequest(undefined, endpoint, 200, indexExpectedHeaders, done); }); }); @@ -695,66 +712,65 @@ describe('Head request on bucket website endpoint', () => { const webConfig = new WebsiteConfigTester('index.html'); const object = { Bucket: bucket, - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/index.html')), + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/index.html')), ContentType: 'text/html', }; - - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })); - await s3.send(new PutBucketPolicyCommand({ Bucket: bucket, - Policy: JSON.stringify({ - Version: '2012-10-17', - Statement: [{ - Sid: 'PublicReadGetObject', - Effect: 'Allow', - Principal: '*', - Action: ['s3:GetObject'], - Resource: [ - `arn:aws:s3:::${bucket}/original_key_file`, - `arn:aws:s3:::${bucket}/original_key_nofile`, - `arn:aws:s3:::${bucket}/file/*`, - `arn:aws:s3:::${bucket}/nofile/*`, + + await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })); + await s3.send( + new PutBucketPolicyCommand({ + Bucket: bucket, + Policy: JSON.stringify({ + Version: '2012-10-17', + Statement: [ + { + Sid: 'PublicReadGetObject', + Effect: 'Allow', + Principal: '*', + Action: ['s3:GetObject'], + Resource: [ + `arn:aws:s3:::${bucket}/original_key_file`, + `arn:aws:s3:::${bucket}/original_key_nofile`, + `arn:aws:s3:::${bucket}/file/*`, + `arn:aws:s3:::${bucket}/nofile/*`, + ], + }, ], - }], - }) - })); - await s3.send(new PutObjectCommand(Object.assign({}, object, - { Key: 'original_key_file/index.html' }))); - await s3.send(new PutObjectCommand(Object.assign({}, object, - { Key: 'file/index.html' }))); - await s3.send(new PutObjectCommand(Object.assign({}, object, - { Key: 'no_access_file/index.html' }))); + }), + }), + ); + await s3.send(new PutObjectCommand(Object.assign({}, object, { Key: 'original_key_file/index.html' }))); + await s3.send(new PutObjectCommand(Object.assign({}, object, { Key: 'file/index.html' }))); + await s3.send(new PutObjectCommand(Object.assign({}, object, { Key: 'no_access_file/index.html' }))); }); afterEach(async () => { - await s3.send(new DeleteObjectCommand({ Bucket: bucket, - Key: 'original_key_file/index.html' })); - await s3.send(new DeleteObjectCommand({ Bucket: bucket, - Key: 'file/index.html' })); - await s3.send(new DeleteObjectCommand({ Bucket: bucket, - Key: 'no_access_file/index.html' })); + await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: 'original_key_file/index.html' })); + await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: 'file/index.html' })); + await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: 'no_access_file/index.html' })); }); it('should redirect 302 with trailing / on folder with index', done => { const expectedHeaders = { - 'location': '/file/', + location: '/file/', 'x-amz-error-code': 'Found', 'x-amz-error-message': 'Resource Found', }; - WebsiteConfigTester.makeHeadRequest(undefined, - `${endpoint}/file`, 302, expectedHeaders, done); + WebsiteConfigTester.makeHeadRequest(undefined, `${endpoint}/file`, 302, expectedHeaders, done); }); - it('should return 404 on original key access without index', - done => { + it('should return 404 on original key access without index', done => { const expectedHeaders = { 'x-amz-error-code': 'NoSuchKey', 'x-amz-error-message': 'The specified key does not exist.', }; - WebsiteConfigTester.makeHeadRequest(undefined, - `${endpoint}/original_key_nofile`, 404, - expectedHeaders, done); + WebsiteConfigTester.makeHeadRequest( + undefined, + `${endpoint}/original_key_nofile`, + 404, + expectedHeaders, + done, + ); }); describe('should return 403', () => { @@ -777,10 +793,15 @@ describe('Head request on bucket website endpoint', () => { 'x-amz-error-code': 'AccessDenied', 'x-amz-error-message': 'Access Denied', }; - WebsiteConfigTester.makeHeadRequest(undefined, - `${endpoint}/${test.key}`, 403, - expectedHeaders, done); - })); + WebsiteConfigTester.makeHeadRequest( + undefined, + `${endpoint}/${test.key}`, + 403, + expectedHeaders, + done, + ); + }), + ); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/object/websiteHeadWithACL.js b/tests/functional/aws-node-sdk/test/object/websiteHeadWithACL.js index 79ecf2d2b5..1ed267396f 100644 --- a/tests/functional/aws-node-sdk/test/object/websiteHeadWithACL.js +++ b/tests/functional/aws-node-sdk/test/object/websiteHeadWithACL.js @@ -12,13 +12,11 @@ const s3 = new S3Client(config); // `127.0.0.1 bucketwebsitetester.s3-website-us-east-1.amazonaws.com` const transport = conf.https ? 'https' : 'http'; -const bucket = process.env.AWS_ON_AIR ? 'awsbucketwebsitetester' : - 'bucketwebsitetester'; -const hostname = process.env.S3_END_TO_END ? - `${bucket}.s3-website-us-east-1.scality.com` : - `${bucket}.s3-website-us-east-1.amazonaws.com`; -const endpoint = process.env.AWS_ON_AIR ? `${transport}://${hostname}` : - `${transport}://${hostname}:8000`; +const bucket = process.env.AWS_ON_AIR ? 'awsbucketwebsitetester' : 'bucketwebsitetester'; +const hostname = process.env.S3_END_TO_END + ? `${bucket}.s3-website-us-east-1.scality.com` + : `${bucket}.s3-website-us-east-1.amazonaws.com`; +const endpoint = process.env.AWS_ON_AIR ? `${transport}://${hostname}` : `${transport}://${hostname}:8000`; const aclEquivalent = { public: ['public-read-write', 'public-read'], @@ -56,84 +54,73 @@ const aclTests = [ result: 'accessDenied', }, { - it: 'should return 403 if public bucket - private index - public ' + - 'error documents', + it: 'should return 403 if public bucket - private index - public ' + 'error documents', bucketACL: 'public', objects: { index: 'private', error: 'private' }, result: 'accessDenied', }, { - it: 'should return 200 if private bucket - public index - ' + - 'public error documents', + it: 'should return 200 if private bucket - public index - ' + 'public error documents', bucketACL: 'private', objects: { index: 'public-read', error: 'private' }, result: 'index', }, { - it: 'should return 200 if public bucket - public index - ' + - 'private error documents', + it: 'should return 200 if public bucket - public index - ' + 'private error documents', bucketACL: 'public', objects: { index: 'public-read', error: 'private' }, result: 'index', }, { - it: 'should return 200 if private bucket - public index - ' + - 'public error documents', + it: 'should return 200 if private bucket - public index - ' + 'public error documents', bucketACL: 'private', objects: { index: 'public-read', error: 'public-read' }, result: 'index', }, { - it: 'should return 200 if public bucket - public index - ' + - 'public error documents', + it: 'should return 200 if public bucket - public index - ' + 'public error documents', bucketACL: 'public', objects: { index: 'public-read', error: 'public-read' }, result: 'index', }, { - it: 'should return 403 AccessDenied if private bucket - ' + - 'without index - public error documents', + it: 'should return 403 AccessDenied if private bucket - ' + 'without index - public error documents', bucketACL: 'private', objects: { error: 'public-read' }, result: 'accessDenied', }, { - it: 'should return 404 if public bucket - without index - ' + - 'public error documents', + it: 'should return 404 if public bucket - without index - ' + 'public error documents', bucketACL: 'public', objects: { error: 'public-read' }, result: 'noSuchKey', }, { - it: 'should return 403 if private bucket - without index - ' + - 'private error documents', + it: 'should return 403 if private bucket - without index - ' + 'private error documents', bucketACL: 'private', objects: { error: 'private' }, result: 'accessDenied', }, { - it: 'should return 404 if public bucket - without index - ' + - 'private error documents', + it: 'should return 404 if public bucket - without index - ' + 'private error documents', bucketACL: 'public', objects: { error: 'private' }, result: 'noSuchKey', }, { - it: 'should return 404 if public bucket - without index - ' + - 'without error documents', + it: 'should return 404 if public bucket - without index - ' + 'without error documents', bucketACL: 'public', - objects: { }, + objects: {}, result: 'noSuchKey', }, { - it: 'should return 403 if private bucket - without index - ' + - 'without error documents', + it: 'should return 403 if private bucket - without index - ' + 'without error documents', bucketACL: 'private', - objects: { }, + objects: {}, result: 'accessDenied', }, ]; @@ -143,33 +130,43 @@ describe('Head request on bucket website endpoint with ACL', () => { aclEquivalent[test.bucketACL].forEach(bucketACL => { describe(`with existing bucket with ${bucketACL} acl`, () => { beforeEach(done => { - WebsiteConfigTester.createPutBucketWebsite(s3, bucket, - bucketACL, test.objects, done); + WebsiteConfigTester.createPutBucketWebsite(s3, bucket, bucketACL, test.objects, done); }); afterEach(done => { - WebsiteConfigTester.deleteObjectsThenBucket(s3, bucket, - test.objects, done); + WebsiteConfigTester.deleteObjectsThenBucket(s3, bucket, test.objects, done); }); it(`${test.it} with no auth credentials sent`, done => { const result = test.result; - WebsiteConfigTester.makeHeadRequest(undefined, endpoint, + WebsiteConfigTester.makeHeadRequest( + undefined, + endpoint, headersACL[result].status, - headersACL[result].expectedHeaders, done); + headersACL[result].expectedHeaders, + done, + ); }); it(`${test.it} even with invalid auth credentials`, done => { const result = test.result; - WebsiteConfigTester.makeHeadRequest('invalid credentials', - endpoint, headersACL[result].status, - headersACL[result].expectedHeaders, done); + WebsiteConfigTester.makeHeadRequest( + 'invalid credentials', + endpoint, + headersACL[result].status, + headersACL[result].expectedHeaders, + done, + ); }); it(`${test.it} even with valid auth credentials`, done => { const result = test.result; - WebsiteConfigTester.makeHeadRequest('valid credentials', - endpoint, headersACL[result].status, - headersACL[result].expectedHeaders, done); + WebsiteConfigTester.makeHeadRequest( + 'valid credentials', + endpoint, + headersACL[result].status, + headersACL[result].expectedHeaders, + done, + ); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/object/websiteRuleMixing.js b/tests/functional/aws-node-sdk/test/object/websiteRuleMixing.js index 71acdda127..5e218c13e7 100644 --- a/tests/functional/aws-node-sdk/test/object/websiteRuleMixing.js +++ b/tests/functional/aws-node-sdk/test/object/websiteRuleMixing.js @@ -21,326 +21,405 @@ const s3 = bucketUtil.s3; // `127.0.0.1 bucketwebsitetester.s3-website-us-east-1.amazonaws.com` const transport = conf.https ? 'https' : 'http'; -const bucket = process.env.AWS_ON_AIR ? 'awsbucketwebsitetester' : - 'bucketwebsitetester'; -const hostname = process.env.S3_END_TO_END ? - `${bucket}.s3-website-us-east-1.scality.com` : - `${bucket}.s3-website-us-east-1.amazonaws.com`; -const endpoint = process.env.AWS_ON_AIR ? `${transport}://${hostname}` : - `${transport}://${hostname}:8000`; -const redirectEndpoint = conf.https ? 'https://www.google.com/' : - 'http://www.google.com/'; - -describe('User visits bucket website endpoint and requests resource ' + -'that has x-amz-website-redirect-location header ::', () => { - beforeEach(async () => await s3.send(new CreateBucketCommand({ Bucket: bucket }))); - - afterEach(async () => await s3.send(new DeleteBucketCommand({ Bucket: bucket }))); - - describe('when x-amz-website-redirect-location: /redirect.html', () => { - beforeEach(async () => { - const webConfig = new WebsiteConfigTester('index.html'); - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })); - await s3.send(new PutObjectCommand({ Bucket: bucket, - Key: 'index.html', - ACL: 'public-read', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/index.html')), - ContentType: 'text/html', - WebsiteRedirectLocation: '/redirect.html' })); - await s3.send(new PutObjectCommand({ Bucket: bucket, - Key: 'redirect.html', - ACL: 'public-read', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/redirect.html')), - ContentType: 'text/html' })); - }); - - afterEach(async () => await bucketUtil.empty(bucket)); - - it('should serve redirect file on GET request', done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: endpoint, - responseType: 'redirect', - redirectUrl: '/redirect.html', - }, done); - }); - - it('should redirect to redirect file on HEAD request', done => { - WebsiteConfigTester.checkHTML({ - method: 'HEAD', - url: endpoint, - responseType: 'redirect', - redirectUrl: '/redirect.html', - }, done); - }); - }); - - describe('when x-amz-website-redirect-location: https://www.google.com', +const bucket = process.env.AWS_ON_AIR ? 'awsbucketwebsitetester' : 'bucketwebsitetester'; +const hostname = process.env.S3_END_TO_END + ? `${bucket}.s3-website-us-east-1.scality.com` + : `${bucket}.s3-website-us-east-1.amazonaws.com`; +const endpoint = process.env.AWS_ON_AIR ? `${transport}://${hostname}` : `${transport}://${hostname}:8000`; +const redirectEndpoint = conf.https ? 'https://www.google.com/' : 'http://www.google.com/'; + +describe( + 'User visits bucket website endpoint and requests resource ' + 'that has x-amz-website-redirect-location header ::', () => { - beforeEach(async () => { - const webConfig = new WebsiteConfigTester('index.html'); - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })); - await s3.send(new PutObjectCommand({ Bucket: bucket, - Key: 'index.html', - ACL: 'public-read', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/index.html')), - ContentType: 'text/html', - WebsiteRedirectLocation: 'https://www.google.com' })); - }); + beforeEach(async () => await s3.send(new CreateBucketCommand({ Bucket: bucket }))); + + afterEach(async () => await s3.send(new DeleteBucketCommand({ Bucket: bucket }))); + + describe('when x-amz-website-redirect-location: /redirect.html', () => { + beforeEach(async () => { + const webConfig = new WebsiteConfigTester('index.html'); + await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: 'index.html', + ACL: 'public-read', + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/index.html')), + ContentType: 'text/html', + WebsiteRedirectLocation: '/redirect.html', + }), + ); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: 'redirect.html', + ACL: 'public-read', + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/redirect.html')), + ContentType: 'text/html', + }), + ); + }); - afterEach(async () => await bucketUtil.empty(bucket)); + afterEach(async () => await bucketUtil.empty(bucket)); + + it('should serve redirect file on GET request', done => { + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: endpoint, + responseType: 'redirect', + redirectUrl: '/redirect.html', + }, + done, + ); + }); - it('should redirect to https://www.google.com', done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: endpoint, - responseType: 'redirect', - redirectUrl: 'https://www.google.com', - }, done); + it('should redirect to redirect file on HEAD request', done => { + WebsiteConfigTester.checkHTML( + { + method: 'HEAD', + url: endpoint, + responseType: 'redirect', + redirectUrl: '/redirect.html', + }, + done, + ); + }); }); - it('should redirect to https://www.google.com on HEAD request', - done => { - WebsiteConfigTester.checkHTML({ - method: 'HEAD', - url: endpoint, - responseType: 'redirect', - redirectUrl: 'https://www.google.com', - }, done); + describe('when x-amz-website-redirect-location: https://www.google.com', () => { + beforeEach(async () => { + const webConfig = new WebsiteConfigTester('index.html'); + await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: 'index.html', + ACL: 'public-read', + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/index.html')), + ContentType: 'text/html', + WebsiteRedirectLocation: 'https://www.google.com', + }), + ); }); - }); - - describe('when key with header is private', () => { - beforeEach(async () => { - const webConfig = new WebsiteConfigTester('index.html'); - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })); - await s3.send(new PutObjectCommand({ Bucket: bucket, - Key: 'index.html', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/index.html')), - ContentType: 'text/html', - WebsiteRedirectLocation: 'https://www.google.com' })); - }); - afterEach(async () => await bucketUtil.empty(bucket)); + afterEach(async () => await bucketUtil.empty(bucket)); + + it('should redirect to https://www.google.com', done => { + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: endpoint, + responseType: 'redirect', + redirectUrl: 'https://www.google.com', + }, + done, + ); + }); - it('should return 403 instead of x-amz-website-redirect-location ' + - 'header location', done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: endpoint, - responseType: '403-access-denied', - }, done); + it('should redirect to https://www.google.com on HEAD request', done => { + WebsiteConfigTester.checkHTML( + { + method: 'HEAD', + url: endpoint, + responseType: 'redirect', + redirectUrl: 'https://www.google.com', + }, + done, + ); + }); }); - it('should return 403 instead of x-amz-website-redirect-location ' + - 'header location on HEAD request', done => { - WebsiteConfigTester.checkHTML({ - method: 'HEAD', - url: endpoint, - responseType: '403-access-denied', - }, done); - }); - }); - - describe('when key with header is private' + - 'and website config has error condition routing rule', () => { - beforeEach(async () => { - const webConfig = new WebsiteConfigTester('index.html'); - const condition = { - HttpErrorCodeReturnedEquals: '403', - }; - const redirect = { - HostName: 'www.google.com', - }; - webConfig.addRoutingRule(redirect, condition); - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })); - await s3.send(new PutObjectCommand({ Bucket: bucket, - Key: 'index.html', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/index.html')), - ContentType: 'text/html', - WebsiteRedirectLocation: '/redirect.html' })); - await s3.send(new PutObjectCommand({ Bucket: bucket, - Key: 'redirect.html', - ACL: 'public-read', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/redirect.html')), - ContentType: 'text/html' })); - }); + describe('when key with header is private', () => { + beforeEach(async () => { + const webConfig = new WebsiteConfigTester('index.html'); + await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: 'index.html', + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/index.html')), + ContentType: 'text/html', + WebsiteRedirectLocation: 'https://www.google.com', + }), + ); + }); - afterEach(async () => await bucketUtil.empty(bucket)); - - it(`should redirect to ${redirectEndpoint} since error 403 ` + - 'occurred instead of x-amz-website-redirect-location header ' + - 'location on GET request', done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: endpoint, - responseType: 'redirect', - redirectUrl: redirectEndpoint, - }, done); - }); + afterEach(async () => await bucketUtil.empty(bucket)); + + it('should return 403 instead of x-amz-website-redirect-location ' + 'header location', done => { + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: endpoint, + responseType: '403-access-denied', + }, + done, + ); + }); - it(`should redirect to ${redirectEndpoint} since error 403 ` + - 'occurred instead of x-amz-website-redirect-location header ' + - 'location on HEAD request', - done => { - WebsiteConfigTester.checkHTML({ - method: 'HEAD', - url: endpoint, - responseType: 'redirect', - redirectUrl: redirectEndpoint, - }, done); - }); - }); - - describe(`with redirect all requests to ${redirectEndpoint}`, () => { - beforeEach(async () => { - const redirectAllTo = { - HostName: 'www.google.com', - }; - const webConfig = new WebsiteConfigTester(null, null, - redirectAllTo); - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })); - await s3.send(new PutObjectCommand({ Bucket: bucket, - Key: 'index.html', - ACL: 'public-read', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/index.html')), - ContentType: 'text/html', - WebsiteRedirectLocation: '/redirect.html' })); + it( + 'should return 403 instead of x-amz-website-redirect-location ' + 'header location on HEAD request', + done => { + WebsiteConfigTester.checkHTML( + { + method: 'HEAD', + url: endpoint, + responseType: '403-access-denied', + }, + done, + ); + }, + ); }); - afterEach(async () => await bucketUtil.empty(bucket)); - - it(`should redirect to ${redirectEndpoint} instead of ` + - 'x-amz-website-redirect-location header location on GET request', - done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: endpoint, - responseType: 'redirect', - redirectUrl: redirectEndpoint, - }, done); - }); + describe('when key with header is private' + 'and website config has error condition routing rule', () => { + beforeEach(async () => { + const webConfig = new WebsiteConfigTester('index.html'); + const condition = { + HttpErrorCodeReturnedEquals: '403', + }; + const redirect = { + HostName: 'www.google.com', + }; + webConfig.addRoutingRule(redirect, condition); + await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: 'index.html', + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/index.html')), + ContentType: 'text/html', + WebsiteRedirectLocation: '/redirect.html', + }), + ); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: 'redirect.html', + ACL: 'public-read', + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/redirect.html')), + ContentType: 'text/html', + }), + ); + }); - it(`should redirect to ${redirectEndpoint} instead of ` + - 'x-amz-website-redirect-location header location on HEAD request', - done => { - WebsiteConfigTester.checkHTML({ - method: 'HEAD', - url: endpoint, - responseType: 'redirect', - redirectUrl: redirectEndpoint, - }, done); + afterEach(async () => await bucketUtil.empty(bucket)); + + it( + `should redirect to ${redirectEndpoint} since error 403 ` + + 'occurred instead of x-amz-website-redirect-location header ' + + 'location on GET request', + done => { + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: endpoint, + responseType: 'redirect', + redirectUrl: redirectEndpoint, + }, + done, + ); + }, + ); + + it( + `should redirect to ${redirectEndpoint} since error 403 ` + + 'occurred instead of x-amz-website-redirect-location header ' + + 'location on HEAD request', + done => { + WebsiteConfigTester.checkHTML( + { + method: 'HEAD', + url: endpoint, + responseType: 'redirect', + redirectUrl: redirectEndpoint, + }, + done, + ); + }, + ); }); - }); - describe('with routing rule redirect to hostname with prefix condition', - () => { - beforeEach(async () => { - const webConfig = new WebsiteConfigTester('index.html'); - const condition = { - KeyPrefixEquals: 'about/', - }; - const redirect = { - HostName: 'www.google.com', - }; - webConfig.addRoutingRule(redirect, condition); - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })); - await s3.send(new PutObjectCommand({ Bucket: bucket, - Key: 'about/index.html', - ACL: 'public-read', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/index.html')), - ContentType: 'text/html', - WebsiteRedirectLocation: '/redirect.html' })); - }); + describe(`with redirect all requests to ${redirectEndpoint}`, () => { + beforeEach(async () => { + const redirectAllTo = { + HostName: 'www.google.com', + }; + const webConfig = new WebsiteConfigTester(null, null, redirectAllTo); + await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: 'index.html', + ACL: 'public-read', + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/index.html')), + ContentType: 'text/html', + WebsiteRedirectLocation: '/redirect.html', + }), + ); + }); - afterEach(async () => await bucketUtil.empty(bucket)); - - it(`should redirect GET request to ${redirectEndpoint}about/ ` + - 'instead of about/ key x-amz-website-redirect-location ' + - 'header location', done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: `${endpoint}/about/`, - responseType: 'redirect', - redirectUrl: `${redirectEndpoint}about/`, - }, done); + afterEach(async () => await bucketUtil.empty(bucket)); + + it( + `should redirect to ${redirectEndpoint} instead of ` + + 'x-amz-website-redirect-location header location on GET request', + done => { + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: endpoint, + responseType: 'redirect', + redirectUrl: redirectEndpoint, + }, + done, + ); + }, + ); + + it( + `should redirect to ${redirectEndpoint} instead of ` + + 'x-amz-website-redirect-location header location on HEAD request', + done => { + WebsiteConfigTester.checkHTML( + { + method: 'HEAD', + url: endpoint, + responseType: 'redirect', + redirectUrl: redirectEndpoint, + }, + done, + ); + }, + ); }); - it(`should redirect HEAD request to ${redirectEndpoint}about ` + - 'instead of about/ key x-amz-website-redirect-location ' + - 'header location', done => { - WebsiteConfigTester.checkHTML({ - method: 'HEAD', - url: `${endpoint}/about/`, - responseType: 'redirect', - redirectUrl: `${redirectEndpoint}about/`, - }, done); - }); - }); - - describe('with routing rule replaceKeyWith', () => { - beforeEach(async () => { - const webConfig = new WebsiteConfigTester('index.html'); - const condition = { - KeyPrefixEquals: 'index.html', - }; - const redirect = { - ReplaceKeyWith: 'redirect.html', - }; - webConfig.addRoutingRule(redirect, condition); - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })); - await s3.send(new PutObjectCommand({ Bucket: bucket, - Key: 'index.html', - ACL: 'public-read', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/index.html')), - ContentType: 'text/html', - WebsiteRedirectLocation: 'https://www.google.com' })); - await s3.send(new PutObjectCommand({ Bucket: bucket, - Key: 'redirect.html', - ACL: 'public-read', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/redirect.html')), - ContentType: 'text/html' })); - }); + describe('with routing rule redirect to hostname with prefix condition', () => { + beforeEach(async () => { + const webConfig = new WebsiteConfigTester('index.html'); + const condition = { + KeyPrefixEquals: 'about/', + }; + const redirect = { + HostName: 'www.google.com', + }; + webConfig.addRoutingRule(redirect, condition); + await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: 'about/index.html', + ACL: 'public-read', + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/index.html')), + ContentType: 'text/html', + WebsiteRedirectLocation: '/redirect.html', + }), + ); + }); - afterEach(async () => await bucketUtil.empty(bucket)); - - it('should replace key instead of redirecting to key ' + - 'x-amz-website-redirect-location header location on GET request', - done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: `${endpoint}/index.html`, - responseType: 'redirect-user', - redirectUrl: `${endpoint}/redirect.html`, - }, done); + afterEach(async () => await bucketUtil.empty(bucket)); + + it( + `should redirect GET request to ${redirectEndpoint}about/ ` + + 'instead of about/ key x-amz-website-redirect-location ' + + 'header location', + done => { + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: `${endpoint}/about/`, + responseType: 'redirect', + redirectUrl: `${redirectEndpoint}about/`, + }, + done, + ); + }, + ); + + it( + `should redirect HEAD request to ${redirectEndpoint}about ` + + 'instead of about/ key x-amz-website-redirect-location ' + + 'header location', + done => { + WebsiteConfigTester.checkHTML( + { + method: 'HEAD', + url: `${endpoint}/about/`, + responseType: 'redirect', + redirectUrl: `${redirectEndpoint}about/`, + }, + done, + ); + }, + ); }); - it('should replace key instead of redirecting to key ' + - 'x-amz-website-redirect-location header location on HEAD request', - done => { - WebsiteConfigTester.checkHTML({ - method: 'HEAD', - url: `${endpoint}/index.html`, - responseType: 'redirect-user', - redirectUrl: `${endpoint}/redirect.html`, - }, done); + describe('with routing rule replaceKeyWith', () => { + beforeEach(async () => { + const webConfig = new WebsiteConfigTester('index.html'); + const condition = { + KeyPrefixEquals: 'index.html', + }; + const redirect = { + ReplaceKeyWith: 'redirect.html', + }; + webConfig.addRoutingRule(redirect, condition); + await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: 'index.html', + ACL: 'public-read', + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/index.html')), + ContentType: 'text/html', + WebsiteRedirectLocation: 'https://www.google.com', + }), + ); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: 'redirect.html', + ACL: 'public-read', + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/redirect.html')), + ContentType: 'text/html', + }), + ); }); - }); -}); + + afterEach(async () => await bucketUtil.empty(bucket)); + + it( + 'should replace key instead of redirecting to key ' + + 'x-amz-website-redirect-location header location on GET request', + done => { + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: `${endpoint}/index.html`, + responseType: 'redirect-user', + redirectUrl: `${endpoint}/redirect.html`, + }, + done, + ); + }, + ); + + it( + 'should replace key instead of redirecting to key ' + + 'x-amz-website-redirect-location header location on HEAD request', + done => { + WebsiteConfigTester.checkHTML( + { + method: 'HEAD', + url: `${endpoint}/index.html`, + responseType: 'redirect-user', + redirectUrl: `${endpoint}/redirect.html`, + }, + done, + ); + }, + ); + }); + }, +); diff --git a/tests/functional/aws-node-sdk/test/quota/tooling.js b/tests/functional/aws-node-sdk/test/quota/tooling.js index e9f616459d..ef1519f81d 100644 --- a/tests/functional/aws-node-sdk/test/quota/tooling.js +++ b/tests/functional/aws-node-sdk/test/quota/tooling.js @@ -41,14 +41,14 @@ const sendRequest = async (method, host, path, body = '', config = null, signing // Get credentials - use same source as S3 client configuration let accessKeyId = config?.accessKey || config?.accessKeyId; let secretAccessKey = config?.secretKey || config?.secretAccessKey; - + // If not provided in config, use getCredentials (matches S3 client credential source) if (!accessKeyId || !secretAccessKey) { const defaultCreds = getCredentials('default'); accessKeyId = accessKeyId || defaultCreds.accessKeyId; secretAccessKey = secretAccessKey || defaultCreds.secretAccessKey; } - + if (!accessKeyId || !secretAccessKey) { throw new Error('Missing accessKeyId or secretAccessKey in config'); } diff --git a/tests/functional/aws-node-sdk/test/rateLimit/client.js b/tests/functional/aws-node-sdk/test/rateLimit/client.js index 083eb6271b..c455c71a42 100644 --- a/tests/functional/aws-node-sdk/test/rateLimit/client.js +++ b/tests/functional/aws-node-sdk/test/rateLimit/client.js @@ -30,7 +30,7 @@ skipIfRateLimitDisabled('RateLimitClient', () => { assert.strictEqual(client.isReady(), true); }); - it('should return true when the client is waiting to connect or the first time', () => { + it('should return true when the client is waiting to connect or the first time', () => { const client = new RateLimitClient(config.localCache); assert.strictEqual(client.isReady(), true); }); diff --git a/tests/functional/aws-node-sdk/test/service/get.js b/tests/functional/aws-node-sdk/test/service/get.js index 4352ab4056..18827567c1 100644 --- a/tests/functional/aws-node-sdk/test/service/get.js +++ b/tests/functional/aws-node-sdk/test/service/get.js @@ -1,29 +1,21 @@ const assert = require('assert'); const tv4 = require('tv4'); const async = require('async'); -const { - S3Client, - ListBucketsCommand, - CreateBucketCommand, - DeleteBucketCommand, -} = require('@aws-sdk/client-s3'); +const { S3Client, ListBucketsCommand, CreateBucketCommand, DeleteBucketCommand } = require('@aws-sdk/client-s3'); const BucketUtility = require('../../lib/utility/bucket-util'); const getConfig = require('../support/config'); const withV4 = require('../support/withV4'); const svcSchema = require('../../schema/service'); -const describeFn = process.env.AWS_ON_AIR - ? describe.skip - : describe; +const describeFn = process.env.AWS_ON_AIR ? describe.skip : describe; async function cleanBucket(bucketUtils, s3, Bucket) { try { await bucketUtils.empty(Bucket, true); await bucketUtils.deleteOne(Bucket); } catch (error) { - process.stdout - .write(`Error emptying and deleting bucket: ${error}\n`); + process.stdout.write(`Error emptying and deleting bucket: ${error}\n`); // ignore the error and continue } } @@ -35,9 +27,9 @@ async function cleanAllBuckets(bucketUtils, s3) { const list = await s3.send(new ListBucketsCommand({})); if (list.Buckets && list.Buckets.length) { - process.stdout - .write(`Found ${list.Buckets.length} buckets to clean:\n${ - JSON.stringify(list.Buckets, null, 2)}\n`); + process.stdout.write( + `Found ${list.Buckets.length} buckets to clean:\n${JSON.stringify(list.Buckets, null, 2)}\n`, + ); // clean sequentially to avoid overloading for (const bucket of list.Buckets) { @@ -51,7 +43,6 @@ describeFn('GET Service - AWS.S3.listBuckets', function getService() { let unauthenticatedBucketUtil; describe('When user is unauthorized', () => { - beforeEach(() => { const config = getConfig('default'); unauthenticatedBucketUtil = new BucketUtility('default', config, true); @@ -87,18 +78,19 @@ describeFn('GET Service - AWS.S3.listBuckets', function getService() { }; }); - it('should return 403 and InvalidAccessKeyId ' + - 'if accessKeyId is invalid', async () => { - const invalidAccess = getConfig('default', - Object.assign({}, + it('should return 403 and InvalidAccessKeyId ' + 'if accessKeyId is invalid', async () => { + const invalidAccess = getConfig( + 'default', + Object.assign( + {}, { credentials: { accessKeyId: 'wrong', secretAccessKey: 'wrong again', }, }, - sigCfg - ) + sigCfg, + ), ); const expectedCode = 'InvalidAccessKeyId'; const expectedStatus = 403; @@ -106,8 +98,7 @@ describeFn('GET Service - AWS.S3.listBuckets', function getService() { await testFn(invalidAccess, expectedCode, expectedStatus); }); - it('should return 403 and SignatureDoesNotMatch ' + - 'if credential is polluted', async () => { + it('should return 403 and SignatureDoesNotMatch ' + 'if credential is polluted', async () => { const pollutedConfig = getConfig('default', sigCfg); pollutedConfig.credentials.secretAccessKey = 'wrong'; @@ -122,82 +113,97 @@ describeFn('GET Service - AWS.S3.listBuckets', function getService() { let bucketUtil; let s3; const bucketsNumber = 1001; - process.stdout - .write(`testing listing with ${bucketsNumber} buckets\n`); - const createdBuckets = Array.from(Array(bucketsNumber).keys()) - .map(i => `getservicebuckets-${i}`); + process.stdout.write(`testing listing with ${bucketsNumber} buckets\n`); + const createdBuckets = Array.from(Array(bucketsNumber).keys()).map(i => `getservicebuckets-${i}`); before(done => { bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; - async.series([ - next => cleanAllBuckets(bucketUtil, s3).then(() => next()).catch(next), - next => - async.eachLimit(createdBuckets, 10, (bucketName, moveOn) => { - s3.send(new CreateBucketCommand({ Bucket: bucketName })) - .then(() => { - if (bucketName.endsWith('000')) { - process.stdout - .write(`creating bucket: ${bucketName}\n`); + async.series( + [ + next => + cleanAllBuckets(bucketUtil, s3) + .then(() => next()) + .catch(next), + next => + async.eachLimit( + createdBuckets, + 10, + (bucketName, moveOn) => { + s3.send(new CreateBucketCommand({ Bucket: bucketName })) + .then(() => { + if (bucketName.endsWith('000')) { + process.stdout.write(`creating bucket: ${bucketName}\n`); + } + moveOn(); + }) + .catch(err => { + moveOn(err); + }); + }, + err => { + if (err) { + process.stdout.write(`err creating buckets: ${err}\n`); + return next(err); } - moveOn(); - }) - .catch(err => { - moveOn(err); - }); - }, - err => { - if (err) { - process.stdout.write(`err creating buckets: ${err}\n`); - return next(err); - } - return next(err); - }) - ], done); + return next(err); + }, + ), + ], + done, + ); }); after(done => { - async.eachLimit(createdBuckets, 10, (bucketName, moveOn) => { - s3.send(new DeleteBucketCommand({ Bucket: bucketName })) - .then(() => { - if (bucketName.endsWith('000')) { - // log to keep ci alive - process.stdout - .write(`deleting bucket: ${bucketName}\n`); - } - moveOn(); - }) - .catch(() => { - moveOn(); - }); - }, - err => { - if (err) { - process.stdout.write(`err deleting buckets: ${err}`); - } - done(err); - }); + async.eachLimit( + createdBuckets, + 10, + (bucketName, moveOn) => { + s3.send(new DeleteBucketCommand({ Bucket: bucketName })) + .then(() => { + if (bucketName.endsWith('000')) { + // log to keep ci alive + process.stdout.write(`deleting bucket: ${bucketName}\n`); + } + moveOn(); + }) + .catch(() => { + moveOn(); + }); + }, + err => { + if (err) { + process.stdout.write(`err deleting buckets: ${err}`); + } + done(err); + }, + ); }); it('should list buckets concurrently', done => { - async.times(20, (n, next) => { - s3.send(new ListBucketsCommand({})) - .then(result => { - // Filter for our test buckets only - const ourBuckets = result.Buckets.filter(bucket => - bucket.Name.startsWith('getservicebuckets-') - ); - assert.equal(ourBuckets.length, - createdBuckets.length, - 'Created buckets are missing in response'); - next(); - }) - .catch(next); - }, - err => { - assert.ifError(err, `error listing buckets: ${err}`); - done(); - }); + async.times( + 20, + (n, next) => { + s3.send(new ListBucketsCommand({})) + .then(result => { + // Filter for our test buckets only + const ourBuckets = result.Buckets.filter(bucket => + bucket.Name.startsWith('getservicebuckets-'), + ); + assert.equal( + ourBuckets.length, + createdBuckets.length, + 'Created buckets are missing in response', + ); + next(); + }) + .catch(next); + }, + err => { + assert.ifError(err, `error listing buckets: ${err}`); + done(); + }, + ); }); it('should list buckets', done => { @@ -212,12 +218,9 @@ describeFn('GET Service - AWS.S3.listBuckets', function getService() { return data; }) .then(data => { - const buckets = data.Buckets.filter(bucket => - createdBuckets.indexOf(bucket.Name) > -1 - ); + const buckets = data.Buckets.filter(bucket => createdBuckets.indexOf(bucket.Name) > -1); - assert.equal(buckets.length, createdBuckets.length, - 'Created buckets are missing in response'); + assert.equal(buckets.length, createdBuckets.length, 'Created buckets are missing in response'); return buckets; }) @@ -225,14 +228,12 @@ describeFn('GET Service - AWS.S3.listBuckets', function getService() { // Sort createdBuckets in alphabetical order createdBuckets.sort(); - const isCorrectOrder = buckets - .reduce( - (prev, bucket, idx) => - prev && bucket.Name === createdBuckets[idx] - , true); + const isCorrectOrder = buckets.reduce( + (prev, bucket, idx) => prev && bucket.Name === createdBuckets[idx], + true, + ); - assert.ok(isCorrectOrder, - 'Not returning created buckets by alphabetically'); + assert.ok(isCorrectOrder, 'Not returning created buckets by alphabetically'); done(); }) .catch(done); @@ -248,14 +249,12 @@ describeFn('GET Service - AWS.S3.listBuckets', function getService() { }); it('should not return other accounts bucket list', done => { - anotherS3.send(new ListBucketsCommand({})) + anotherS3 + .send(new ListBucketsCommand({})) .then(data => { - const hasSameBuckets = data.Buckets - .filter(filterFn) - .length; + const hasSameBuckets = data.Buckets.filter(filterFn).length; - assert.strictEqual(hasSameBuckets, 0, - 'It has other buddies bucket'); + assert.strictEqual(hasSameBuckets, 0, 'It has other buddies bucket'); done(); }) .catch(done); diff --git a/tests/functional/aws-node-sdk/test/support/awsConfig.js b/tests/functional/aws-node-sdk/test/support/awsConfig.js index d19ab338c0..d6e02734e6 100644 --- a/tests/functional/aws-node-sdk/test/support/awsConfig.js +++ b/tests/functional/aws-node-sdk/test/support/awsConfig.js @@ -17,14 +17,14 @@ function getAwsCredentials(profile, credFile = '/.aws/credentials') { // Parse the INI file manually for synchronous access const content = fs.readFileSync(filename, 'utf-8'); const profileMatch = content.match(new RegExp(`\\[${profile}\\][\\s\\S]*?(?=\\n\\[|$)`)); - + if (!profileMatch) { throw new Error(`Profile "${profile}" not found in ${filename}`); } - + const accessKeyMatch = profileMatch[0].match(/aws_access_key_id\s*=\s*(.+)/); const secretKeyMatch = profileMatch[0].match(/aws_secret_access_key\s*=\s*(.+)/); - + if (!accessKeyMatch || !secretKeyMatch) { throw new Error(`Missing credentials in profile "${profile}"`); } @@ -36,25 +36,30 @@ function getAwsCredentials(profile, credFile = '/.aws/credentials') { } function getRealAwsConfig(location) { - const { awsEndpoint, gcpEndpoint, credentialsProfile, - credentials: locCredentials, bucketName, mpuBucketName, pathStyle } = - config.locationConstraints[location].details; + const { + awsEndpoint, + gcpEndpoint, + credentialsProfile, + credentials: locCredentials, + bucketName, + mpuBucketName, + pathStyle, + } = config.locationConstraints[location].details; const useHTTPS = config.locationConstraints[location].details.https; const proto = useHTTPS ? 'https' : 'http'; const isGcp = config.locationConstraints[location].type === 'gcp'; const params = { region: 'us-east-1', - endpoint: gcpEndpoint ? - `${proto}://${gcpEndpoint}` : `${proto}://${awsEndpoint}`, + endpoint: gcpEndpoint ? `${proto}://${gcpEndpoint}` : `${proto}://${awsEndpoint}`, }; - + if (isGcp) { params.disableS3ExpressSessionAuth = true; params.useGlobalEndpoint = false; params.s3DisableBodySigning = true; params.mainBucket = bucketName; params.mpuBucket = mpuBucketName; - } + } if (useHTTPS) { params.requestHandler = { httpsAgent: new https.Agent({ keepAlive: true }), @@ -64,25 +69,26 @@ function getRealAwsConfig(location) { httpAgent: new http.Agent({ keepAlive: true }), }; } - + if (pathStyle) { params.forcePathStyle = true; } - + if (!useHTTPS) { params.sslEnabled = false; } - + if (credentialsProfile) { const credentials = getAwsCredentials(credentialsProfile, '/.aws/credentials'); params.credentials = credentials; - + if (isGcp) { return { s3Params: params, bucketName, mpuBucket: mpuBucketName || bucketName, - credentials: { // For raw HTTP requests (GCP format) + credentials: { + // For raw HTTP requests (GCP format) accessKey: credentials.accessKeyId, secretKey: credentials.secretAccessKey, }, @@ -90,11 +96,11 @@ function getRealAwsConfig(location) { } return params; } - params.credentials = { + params.credentials = { accessKeyId: locCredentials.accessKey, secretAccessKey: locCredentials.secretKey, }; - + // For GCP with plain credentials, return nested structure if (isGcp) { return { @@ -113,7 +119,7 @@ function getRealAwsConfig(location) { }, }; } - + return params; } diff --git a/tests/functional/aws-node-sdk/test/support/objectConfigs.js b/tests/functional/aws-node-sdk/test/support/objectConfigs.js index ed2e9cb0bf..cf4324dbdf 100644 --- a/tests/functional/aws-node-sdk/test/support/objectConfigs.js +++ b/tests/functional/aws-node-sdk/test/support/objectConfigs.js @@ -10,9 +10,8 @@ const canonicalObjectConfig = { invalidPartNumbers: [-1, 0, maximumAllowedPartCount + 1], signature: 'for canonical object', meta: { - computeTotalSize: (partNumbers, bodySize) => partNumbers.reduce((total, current) => - total + bodySize + current + 1 - , 0), + computeTotalSize: (partNumbers, bodySize) => + partNumbers.reduce((total, current) => total + bodySize + current + 1, 0), objectIsEmpty: false, }, }; @@ -32,9 +31,6 @@ const emptyObjectConfig = { }, }; -const objectConfigs = [ - canonicalObjectConfig, - emptyObjectConfig, -]; +const objectConfigs = [canonicalObjectConfig, emptyObjectConfig]; module.exports = objectConfigs; diff --git a/tests/functional/aws-node-sdk/test/support/withV4.js b/tests/functional/aws-node-sdk/test/support/withV4.js index e875ed68f2..03dd53c3f0 100644 --- a/tests/functional/aws-node-sdk/test/support/withV4.js +++ b/tests/functional/aws-node-sdk/test/support/withV4.js @@ -12,11 +12,13 @@ function withV4(testFn) { config = {}; } - describe(`With ${version} signature`, (cfg => - function tcWrap() { - testFn.call(this, cfg); - } - )(config)); + describe( + `With ${version} signature`, + (cfg => + function tcWrap() { + testFn.call(this, cfg); + })(config), + ); }); } diff --git a/tests/functional/aws-node-sdk/test/utils/init.js b/tests/functional/aws-node-sdk/test/utils/init.js index 1cfd6a67a8..c39fe7b4c0 100644 --- a/tests/functional/aws-node-sdk/test/utils/init.js +++ b/tests/functional/aws-node-sdk/test/utils/init.js @@ -4,12 +4,9 @@ const metadata = require('../../../../../lib/metadata/wrapper'); const { config } = require('../../../../../lib/Config'); const { DummyRequestLogger } = require('../../../../unit/helpers'); const log = new DummyRequestLogger(); -const nonVersionedObjId = - versionIdUtils.getInfVid(config.replicationGroupId); +const nonVersionedObjId = versionIdUtils.getInfVid(config.replicationGroupId); -const { - LOCATION_NAME_DMF, -} = require('../../../../constants'); +const { LOCATION_NAME_DMF } = require('../../../../constants'); const isMetadataOrFile = ['file', 'scality'].includes(config.backends.metadata); /** @@ -57,13 +54,11 @@ function initMetadata(cb) { } function getMetadata(bucketName, objectName, versionId, cb) { - const promise = new Promise((resolve, reject) => metadata.getObjectMD( - bucketName, - objectName, - { versionId: decodeVersionId(versionId) }, - log, - (err, data) => (err ? reject(err) : resolve(data)), - )); + const promise = new Promise((resolve, reject) => + metadata.getObjectMD(bucketName, objectName, { versionId: decodeVersionId(versionId) }, log, (err, data) => + err ? reject(err) : resolve(data), + ), + ); return cb ? promise.then(res => cb(null, res), cb) : promise; } @@ -80,14 +75,11 @@ function fakeMetadataTransition(bucketName, objectName, versionId, cb) { const promise = (async () => { const objMD = await getMetadata(bucketName, objectName, versionId); objMD['x-amz-scal-transition-in-progress'] = true; - await new Promise((resolve, reject) => metadata.putObjectMD( - bucketName, - objectName, - objMD, - { versionId: decodeVersionId(versionId) }, - log, - err => (err ? reject(err) : resolve()), - )); + await new Promise((resolve, reject) => + metadata.putObjectMD(bucketName, objectName, objMD, { versionId: decodeVersionId(versionId) }, log, err => + err ? reject(err) : resolve(), + ), + ); })(); return cb ? promise.then(() => cb(), cb) : promise; } @@ -108,14 +100,11 @@ function fakeMetadataArchive(bucketName, objectName, versionId, archive, cb) { objMD['x-amz-storage-class'] = LOCATION_NAME_DMF; objMD.dataStoreName = LOCATION_NAME_DMF; objMD.archive = archive; - await new Promise((resolve, reject) => metadata.putObjectMD( - bucketName, - objectName, - objMD, - { versionId: decodeVersionId(versionId) }, - log, - err => (err ? reject(err) : resolve()), - )); + await new Promise((resolve, reject) => + metadata.putObjectMD(bucketName, objectName, objMD, { versionId: decodeVersionId(versionId) }, log, err => + err ? reject(err) : resolve(), + ), + ); })(); return cb ? promise.then(() => cb(), cb) : promise; } diff --git a/tests/functional/aws-node-sdk/test/versioning/bucketDelete.js b/tests/functional/aws-node-sdk/test/versioning/bucketDelete.js index 90dc8315d6..0dc39ac5d7 100644 --- a/tests/functional/aws-node-sdk/test/versioning/bucketDelete.js +++ b/tests/functional/aws-node-sdk/test/versioning/bucketDelete.js @@ -15,7 +15,6 @@ const { removeAllVersions } = require('../../lib/utility/versioning-util.js'); const bucketName = `versioning-bucket-${Date.now()}`; const key = 'anObject'; - function checkError(err, code) { assert.notEqual(err, null, 'Expected failure but got success'); assert.strictEqual(err.Code, code); @@ -29,12 +28,14 @@ describe('aws-node-sdk test delete bucket', () => { // setup test beforeEach(async () => { await s3.send(new CreateBucketCommand({ Bucket: bucketName })); - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: { - Status: 'Enabled', - }, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: { + Status: 'Enabled', + }, + }), + ); }); // empty and delete bucket after testing if bucket exists @@ -43,48 +44,53 @@ describe('aws-node-sdk test delete bucket', () => { if (err?.name === 'NoSuchBucket') { return done(); } - return s3.send(new DeleteBucketCommand({ Bucket: bucketName })) - .then(() => done()).catch(err => { - if (err.name === 'NoSuchBucket') { - return done(); - } - return done(err); - }); + return s3 + .send(new DeleteBucketCommand({ Bucket: bucketName })) + .then(() => done()) + .catch(err => { + if (err.name === 'NoSuchBucket') { + return done(); + } + return done(err); + }); }); }); - it('should be able to delete empty bucket with version enabled', - async () => { + it('should be able to delete empty bucket with version enabled', async () => { await s3.send(new DeleteBucketCommand({ Bucket: bucketName })); }); - it('should return error 409 BucketNotEmpty if trying to delete bucket' + - ' containing delete marker', async () => { - await s3.send(new DeleteObjectCommand({ Bucket: bucketName, Key: key })); - - try { - await s3.send(new DeleteBucketCommand({ Bucket: bucketName })); - assert.fail('Expected BucketNotEmpty error but got success'); - } catch (err) { - checkError(err, 'BucketNotEmpty'); - } - }); + it( + 'should return error 409 BucketNotEmpty if trying to delete bucket' + ' containing delete marker', + async () => { + await s3.send(new DeleteObjectCommand({ Bucket: bucketName, Key: key })); - it('should return error 409 BucketNotEmpty if trying to delete bucket' + - ' containing version and delete marker', async () => { - await s3.send(new PutObjectCommand({ Bucket: bucketName, Key: key })); - await s3.send(new DeleteObjectCommand({ Bucket: bucketName, Key: key })); - - try { - await s3.send(new DeleteBucketCommand({ Bucket: bucketName })); - assert.fail('Expected BucketNotEmpty error but got success'); - } catch (err) { - checkError(err, 'BucketNotEmpty'); - } - }); + try { + await s3.send(new DeleteBucketCommand({ Bucket: bucketName })); + assert.fail('Expected BucketNotEmpty error but got success'); + } catch (err) { + checkError(err, 'BucketNotEmpty'); + } + }, + ); + + it( + 'should return error 409 BucketNotEmpty if trying to delete bucket' + + ' containing version and delete marker', + async () => { + await s3.send(new PutObjectCommand({ Bucket: bucketName, Key: key })); + await s3.send(new DeleteObjectCommand({ Bucket: bucketName, Key: key })); + + try { + await s3.send(new DeleteBucketCommand({ Bucket: bucketName })); + assert.fail('Expected BucketNotEmpty error but got success'); + } catch (err) { + checkError(err, 'BucketNotEmpty'); + } + }, + ); - it('should return error 404 NoSuchBucket if the bucket name is invalid', - async () => { + it('should return error 404 NoSuchBucket if the bucket name is invalid', async () => { try { await s3.send(new DeleteBucketCommand({ Bucket: 'bucketA' })); assert.fail('Expected NoSuchBucket error but got success'); diff --git a/tests/functional/aws-node-sdk/test/versioning/legacyNullVersionCompat.js b/tests/functional/aws-node-sdk/test/versioning/legacyNullVersionCompat.js index b773f77d77..d947e52de9 100644 --- a/tests/functional/aws-node-sdk/test/versioning/legacyNullVersionCompat.js +++ b/tests/functional/aws-node-sdk/test/versioning/legacyNullVersionCompat.js @@ -16,10 +16,7 @@ const { const BucketUtility = require('../../lib/utility/bucket-util'); -const { - removeAllVersions, - versioningEnabled, -} = require('../../lib/utility/versioning-util.js'); +const { removeAllVersions, versioningEnabled } = require('../../lib/utility/versioning-util.js'); // This series of tests can only be enabled on an environment that has // two Cloudserver instances, with one of them in null version @@ -29,8 +26,9 @@ const { // combination of Cloudserver requests to bucketd and the behavior of // bucketd based on those requests. -const describeSkipIfNotExplicitlyEnabled = - process.env.ENABLE_LEGACY_NULL_VERSION_COMPAT_TESTS ? describe : describe.skip; +const describeSkipIfNotExplicitlyEnabled = process.env.ENABLE_LEGACY_NULL_VERSION_COMPAT_TESTS + ? describe + : describe.skip; describeSkipIfNotExplicitlyEnabled('legacy null version compatibility tests', () => { const bucketUtilCompat = new BucketUtility('default', { @@ -46,25 +44,46 @@ describeSkipIfNotExplicitlyEnabled('legacy null version compatibility tests', () // master and no "isNull2" metadata attribute), by using the // Cloudserver endpoint that is configured with null version // compatibility mode enabled. - beforeEach(done => async.series([ - next => s3Compat.send(new CreateBucketCommand({ - Bucket: bucket, - }), next), - next => s3Compat.send(new PutObjectCommand({ - Bucket: bucket, - Key: 'obj', - Body: 'nullbody', - }), next), - next => s3Compat.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningEnabled, - }), next), - next => s3Compat.send(new PutObjectCommand({ - Bucket: bucket, - Key: 'obj', - Body: 'versionedbody', - }), next), - ], done)); + beforeEach(done => + async.series( + [ + next => + s3Compat.send( + new CreateBucketCommand({ + Bucket: bucket, + }), + next, + ), + next => + s3Compat.send( + new PutObjectCommand({ + Bucket: bucket, + Key: 'obj', + Body: 'nullbody', + }), + next, + ), + next => + s3Compat.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningEnabled, + }), + next, + ), + next => + s3Compat.send( + new PutObjectCommand({ + Bucket: bucket, + Key: 'obj', + Body: 'versionedbody', + }), + next, + ), + ], + done, + ), + ); afterEach(done => { removeAllVersions({ Bucket: bucket }, err => { @@ -76,37 +95,56 @@ describeSkipIfNotExplicitlyEnabled('legacy null version compatibility tests', () }); it('updating ACL of legacy null version with non-compat cloudserver', done => { - async.series([ - next => s3.send(new PutObjectAclCommand({ - Bucket: bucket, - Key: 'obj', - VersionId: 'null', - ACL: 'public-read', - }), next), - next => s3.send(new GetObjectAclCommand({ - Bucket: bucket, - Key: 'obj', - VersionId: 'null', - }), (err, acl) => { - assert.ifError(err); - // check that we fetched the updated null version - assert.strictEqual(acl.Grants.length, 2); - next(); - }), - next => s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: 'obj', - VersionId: 'null', - }), next), - next => s3.send(new ListObjectVersionsCommand({ - Bucket: bucket, - }), (err, listing) => { - assert.ifError(err); - // check that the null version has been correctly deleted - assert(listing.Versions.every(version => version.VersionId !== 'null')); - next(); - }), - ], done); + async.series( + [ + next => + s3.send( + new PutObjectAclCommand({ + Bucket: bucket, + Key: 'obj', + VersionId: 'null', + ACL: 'public-read', + }), + next, + ), + next => + s3.send( + new GetObjectAclCommand({ + Bucket: bucket, + Key: 'obj', + VersionId: 'null', + }), + (err, acl) => { + assert.ifError(err); + // check that we fetched the updated null version + assert.strictEqual(acl.Grants.length, 2); + next(); + }, + ), + next => + s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: 'obj', + VersionId: 'null', + }), + next, + ), + next => + s3.send( + new ListObjectVersionsCommand({ + Bucket: bucket, + }), + (err, listing) => { + assert.ifError(err); + // check that the null version has been correctly deleted + assert(listing.Versions.every(version => version.VersionId !== 'null')); + next(); + }, + ), + ], + done, + ); }); it('updating tags of legacy null version with non-compat cloudserver', done => { @@ -116,54 +154,81 @@ describeSkipIfNotExplicitlyEnabled('legacy null version compatibility tests', () Value: 'newtagvalue', }, ]; - async.series([ - next => s3.send(new PutObjectTaggingCommand({ - Bucket: bucket, - Key: 'obj', - VersionId: 'null', - Tagging: { - TagSet: tagSet, - }, - }), next), - next => s3.send(new GetObjectTaggingCommand({ - Bucket: bucket, - Key: 'obj', - VersionId: 'null', - }), (err, tagging) => { - assert.ifError(err); - assert.deepStrictEqual(tagging.TagSet, tagSet); - next(); - }), - next => s3.send(new DeleteObjectTaggingCommand({ - Bucket: bucket, - Key: 'obj', - VersionId: 'null', - }), err => { - assert.ifError(err); - next(); - }), - next => s3.send(new GetObjectTaggingCommand({ - Bucket: bucket, - Key: 'obj', - VersionId: 'null', - }), (err, tagging) => { - assert.ifError(err); - assert.deepStrictEqual(tagging.TagSet, []); - next(); - }), - next => s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: 'obj', - VersionId: 'null', - }), next), - next => s3.send(new ListObjectVersionsCommand({ - Bucket: bucket, - }), (err, listing) => { - assert.ifError(err); - // check that the null version has been correctly deleted - assert(listing.Versions.every(version => version.VersionId !== 'null')); - next(); - }), - ], done); + async.series( + [ + next => + s3.send( + new PutObjectTaggingCommand({ + Bucket: bucket, + Key: 'obj', + VersionId: 'null', + Tagging: { + TagSet: tagSet, + }, + }), + next, + ), + next => + s3.send( + new GetObjectTaggingCommand({ + Bucket: bucket, + Key: 'obj', + VersionId: 'null', + }), + (err, tagging) => { + assert.ifError(err); + assert.deepStrictEqual(tagging.TagSet, tagSet); + next(); + }, + ), + next => + s3.send( + new DeleteObjectTaggingCommand({ + Bucket: bucket, + Key: 'obj', + VersionId: 'null', + }), + err => { + assert.ifError(err); + next(); + }, + ), + next => + s3.send( + new GetObjectTaggingCommand({ + Bucket: bucket, + Key: 'obj', + VersionId: 'null', + }), + (err, tagging) => { + assert.ifError(err); + assert.deepStrictEqual(tagging.TagSet, []); + next(); + }, + ), + next => + s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: 'obj', + VersionId: 'null', + }), + next, + ), + next => + s3.send( + new ListObjectVersionsCommand({ + Bucket: bucket, + }), + (err, listing) => { + assert.ifError(err); + // check that the null version has been correctly deleted + assert(listing.Versions.every(version => version.VersionId !== 'null')); + next(); + }, + ), + ], + done, + ); }); }); diff --git a/tests/functional/aws-node-sdk/test/versioning/listObjectMasterVersions.js b/tests/functional/aws-node-sdk/test/versioning/listObjectMasterVersions.js index d8a08ba99f..46fb7db8db 100644 --- a/tests/functional/aws-node-sdk/test/versioning/listObjectMasterVersions.js +++ b/tests/functional/aws-node-sdk/test/versioning/listObjectMasterVersions.js @@ -16,22 +16,13 @@ const { removeAllVersions } = require('../../lib/utility/versioning-util'); const bucket = `versioning-bucket-${Date.now()}`; const itSkipIfE2E = process.env.S3_END_TO_END ? it.skip : it; - function _assertResultElements(entry) { - const elements = [ - 'LastModified', - 'ETag', - 'Size', - 'Owner', - 'StorageClass', - ]; + const elements = ['LastModified', 'ETag', 'Size', 'Owner', 'StorageClass']; elements.forEach(elem => { - assert.notStrictEqual(entry[elem], undefined, - `Expected ${elem} in result but did not find it`); + assert.notStrictEqual(entry[elem], undefined, `Expected ${elem} in result but did not find it`); if (elem === 'Owner') { assert(entry.Owner.ID, 'Expected Owner ID but did not find it'); - assert(entry.Owner.DisplayName, - 'Expected Owner DisplayName but did not find it'); + assert(entry.Owner.DisplayName, 'Expected Owner DisplayName but did not find it'); } }); } @@ -53,8 +44,10 @@ describe('listObject - Delimiter master', function testSuite() { if (err) { return done(err); } - return s3.send(new DeleteBucketCommand({ Bucket: bucket })) - .then(() => done()).catch(done); + return s3 + .send(new DeleteBucketCommand({ Bucket: bucket })) + .then(() => done()) + .catch(done); }); }); @@ -107,17 +100,21 @@ describe('listObject - Delimiter master', function testSuite() { } if (obj.value === null) { - const result = await s3.send(new DeleteObjectCommand({ + const result = await s3.send( + new DeleteObjectCommand({ Bucket: bucket, Key: obj.name, - })); - assert.strictEqual(result.DeleteMarker, true, 'Expected delete marker to be true'); + }), + ); + assert.strictEqual(result.DeleteMarker, true, 'Expected delete marker to be true'); } else { - await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: obj.name, - Body: obj.value, - })); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: obj.name, + Body: obj.value, + }), + ); } } }); @@ -168,11 +165,7 @@ describe('listObject - Delimiter master', function testSuite() { { name: 'with maxKeys', params: { MaxKeys: 3 }, - expectedResult: [ - 'Pâtisserie=中文-español-English', - 'notes/spring/1.txt', - 'notes/spring/march/1.txt', - ], + expectedResult: ['Pâtisserie=中文-español-English', 'notes/spring/1.txt', 'notes/spring/march/1.txt'], commonPrefix: [], isTruncated: true, nextMarker: undefined, @@ -198,9 +191,7 @@ describe('listObject - Delimiter master', function testSuite() { { name: 'with delimiter', params: { Delimiter: '/' }, - expectedResult: [ - 'Pâtisserie=中文-español-English', - ], + expectedResult: ['Pâtisserie=中文-español-English'], commonPrefix: ['notes/'], isTruncated: false, nextMarker: undefined, @@ -235,15 +226,8 @@ describe('listObject - Delimiter master', function testSuite() { { name: 'delimiter and prefix (related to #147)', params: { Delimiter: '/', Prefix: 'notes/' }, - expectedResult: [ - 'notes/year.txt', - 'notes/yore.rs', - ], - commonPrefix: [ - 'notes/spring/', - 'notes/summer/', - 'notes/zaphod/', - ], + expectedResult: ['notes/year.txt', 'notes/yore.rs'], + commonPrefix: ['notes/spring/', 'notes/summer/', 'notes/zaphod/'], isTruncated: false, nextMarker: undefined, }, @@ -329,20 +313,16 @@ describe('listObject - Delimiter master', function testSuite() { runTest(test.name, async () => { const expectedResult = test.expectedResult; const res = await s3.send(new ListObjectsCommand(Object.assign({ Bucket: bucket }, test.params))); - + res.Contents?.forEach(result => { - if (!expectedResult - .find(key => key === result.Key)) { - throw new Error('listing fail, ' + - `unexpected key ${result.Key}`); + if (!expectedResult.find(key => key === result.Key)) { + throw new Error('listing fail, ' + `unexpected key ${result.Key}`); } _assertResultElements(result); }); res.CommonPrefixes?.forEach(cp => { - if (!test.commonPrefix - .find(item => item === cp.Prefix)) { - throw new Error('listing fail, ' + - `unexpected prefix ${cp.Prefix}`); + if (!test.commonPrefix.find(item => item === cp.Prefix)) { + throw new Error('listing fail, ' + `unexpected prefix ${cp.Prefix}`); } }); assert.strictEqual(res.IsTruncated, test.isTruncated); diff --git a/tests/functional/aws-node-sdk/test/versioning/listObjectVersions.js b/tests/functional/aws-node-sdk/test/versioning/listObjectVersions.js index 1685ec5413..102b220429 100644 --- a/tests/functional/aws-node-sdk/test/versioning/listObjectVersions.js +++ b/tests/functional/aws-node-sdk/test/versioning/listObjectVersions.js @@ -17,34 +17,21 @@ const { removeAllVersions } = require('../../lib/utility/versioning-util'); const bucket = `versioning-bucket-${Date.now()}`; const removeAllVersionsAsync = promisify(removeAllVersions); -const resultElements = [ - 'VersionId', - 'IsLatest', - 'LastModified', - 'Owner', -]; -const versionResultElements = [ - 'ETag', - 'Size', - 'StorageClass', -]; +const resultElements = ['VersionId', 'IsLatest', 'LastModified', 'Owner']; +const versionResultElements = ['ETag', 'Size', 'StorageClass']; function _assertResultElements(entry, type) { - const elements = type === 'DeleteMarker' ? resultElements : - resultElements.concat(versionResultElements); + const elements = type === 'DeleteMarker' ? resultElements : resultElements.concat(versionResultElements); elements.forEach(elem => { - assert.notStrictEqual(entry[elem], undefined, - `Expected ${elem} in result but did not find it`); + assert.notStrictEqual(entry[elem], undefined, `Expected ${elem} in result but did not find it`); if (elem === 'Owner') { assert(entry.Owner.ID, 'Expected Owner ID but did not find it'); - assert(entry.Owner.DisplayName, - 'Expected Owner DisplayName but did not find it'); + assert(entry.Owner.DisplayName, 'Expected Owner DisplayName but did not find it'); } }); } - describe('listObject - Delimiter version', function testSuite() { this.timeout(600000); @@ -91,33 +78,41 @@ describe('listObject - Delimiter version', function testSuite() { for (const obj of objects) { // Toggle bucket versioning state according to the original logic if (!versioning && obj.isNull !== true) { - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Enabled' }, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ); versioning = true; } else if (versioning && obj.isNull === true) { - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Suspended' }, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Suspended' }, + }), + ); versioning = false; } if (obj.value === null) { // Create a delete marker, capture headers as in original test - const delRes = await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: obj.name, - })); + const delRes = await s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: obj.name, + }), + ); assert.strictEqual(String(delRes.DeleteMarker), 'true'); obj.versionId = delRes.VersionId; } else { - const putRes = await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: obj.name, - Body: obj.value, - })); + const putRes = await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: obj.name, + Body: obj.value, + }), + ); obj.versionId = putRes.VersionId || 'null'; } } @@ -171,11 +166,7 @@ describe('listObject - Delimiter version', function testSuite() { { name: 'with maxKeys', params: { MaxKeys: 3 }, - expectedResult: [ - objects[4], - objects[5], - objects[8], - ], + expectedResult: [objects[4], objects[5], objects[8]], commonPrefix: [], isTruncated: true, nextKeyMarker: objects[8].name, @@ -202,8 +193,7 @@ describe('listObject - Delimiter version', function testSuite() { { name: 'with long delimiter', params: { Delimiter: 'notes/summer' }, - expectedResult: objects.filter(obj => - obj.name.indexOf('notes/summer') < 0), + expectedResult: objects.filter(obj => obj.name.indexOf('notes/summer') < 0), commonPrefix: ['notes/summer'], isTruncated: false, nextKeyMarker: undefined, @@ -225,15 +215,8 @@ describe('listObject - Delimiter version', function testSuite() { { name: 'delimiter and prefix (related to #147)', params: { Delimiter: '/', Prefix: 'notes/' }, - expectedResult: [ - objects[1], - objects[2], - ], - commonPrefix: [ - 'notes/spring/', - 'notes/summer/', - 'notes/zaphod/', - ], + expectedResult: [objects[1], objects[2]], + commonPrefix: ['notes/spring/', 'notes/summer/', 'notes/zaphod/'], isTruncated: false, nextKeyMarker: undefined, nextVersionIdMarker: undefined, @@ -324,59 +307,52 @@ describe('listObject - Delimiter version', function testSuite() { ].forEach(test => { it(test.name, async () => { const expectedResult = test.expectedResult; - const res = await s3.send(new ListObjectVersionsCommand({ - Bucket: bucket, - ...test.params, - })); + const res = await s3.send( + new ListObjectVersionsCommand({ + Bucket: bucket, + ...test.params, + }), + ); (res.Versions || []).forEach(result => { - const item = expectedResult.find(obj => ( - obj.name === result.Key - && obj.versionId === result.VersionId - && obj.value !== null - )); + const item = expectedResult.find( + obj => obj.name === result.Key && obj.versionId === result.VersionId && obj.value !== null, + ); if (!item) { - throw new Error('listing fail, ' - + `unexpected key ${result.Key} ` - + `with version ${result.VersionId}`); + throw new Error( + 'listing fail, ' + `unexpected key ${result.Key} ` + `with version ${result.VersionId}`, + ); } _assertResultElements(result, 'Version'); }); (res.DeleteMarkers || []).forEach(result => { - const item = expectedResult.find(obj => ( - obj.name === result.Key - && obj.versionId === result.VersionId - && obj.value === null - )); + const item = expectedResult.find( + obj => obj.name === result.Key && obj.versionId === result.VersionId && obj.value === null, + ); if (!item) { - throw new Error('listing fail, ' - + `unexpected key ${result.Key} ` - + `with version ${result.VersionId}`); + throw new Error( + 'listing fail, ' + `unexpected key ${result.Key} ` + `with version ${result.VersionId}`, + ); } _assertResultElements(result, 'DeleteMarker'); }); (res.CommonPrefixes || []).forEach(cp => { - if (!test.commonPrefix.find( - item => item === cp.Prefix, - )) { - throw new Error('listing fail, ' - + `unexpected prefix ${cp.Prefix}`); + if (!test.commonPrefix.find(item => item === cp.Prefix)) { + throw new Error('listing fail, ' + `unexpected prefix ${cp.Prefix}`); } }); assert.strictEqual(res.IsTruncated, test.isTruncated); - assert.strictEqual(res.NextKeyMarker, - test.nextKeyMarker); + assert.strictEqual(res.NextKeyMarker, test.nextKeyMarker); if (!test.nextVersionIdMarker) { // eslint-disable-next-line no-param-reassign test.nextVersionIdMarker = {}; } - assert.strictEqual(res.NextVersionIdMarker, - test.nextVersionIdMarker.versionId); + assert.strictEqual(res.NextVersionIdMarker, test.nextVersionIdMarker.versionId); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/versioning/multiObjectDelete.js b/tests/functional/aws-node-sdk/test/versioning/multiObjectDelete.js index bf5d43d09e..8c50438e83 100644 --- a/tests/functional/aws-node-sdk/test/versioning/multiObjectDelete.js +++ b/tests/functional/aws-node-sdk/test/versioning/multiObjectDelete.js @@ -17,9 +17,9 @@ const bucketName = `multi-object-delete-${Date.now()}`; const key = 'key'; // formats differ for AWS and S3, use respective sample ids to obtain // correct error response in tests -const nonExistingId = process.env.AWS_ON_AIR ? - 'MhhyTHhmZ4cxSi4Y9SMe5P7UJAz7HLJ9' : - '3939393939393939393936493939393939393939756e6437'; +const nonExistingId = process.env.AWS_ON_AIR + ? 'MhhyTHhmZ4cxSi4Y9SMe5P7UJAz7HLJ9' + : '3939393939393939393936493939393939393939756e6437'; function sortList(list) { return list.sort((a, b) => { @@ -33,7 +33,6 @@ function sortList(list) { }); } - describe('Multi-Object Versioning Delete Success', function success() { this.timeout(360000); @@ -44,12 +43,14 @@ describe('Multi-Object Versioning Delete Success', function success() { beforeEach(async () => { await s3.send(new CreateBucketCommand({ Bucket: bucketName })); - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: { - Status: 'Enabled', - }, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: { + Status: 'Enabled', + }, + }), + ); const objects = []; for (let i = 1; i < 1001; i++) { @@ -65,8 +66,9 @@ describe('Multi-Object Versioning Delete Success', function success() { return await s3.send(new PutObjectCommand(params)); } catch (err) { if (attempt < 3) { - process.stdout.write(`Retrying PutObject ${params.Key} ` - + `(attempt ${attempt + 1}/3): ${err}\n`); + process.stdout.write( + `Retrying PutObject ${params.Key} ` + `(attempt ${attempt + 1}/3): ${err}\n`, + ); return putWithRetry(params, attempt + 1); } throw err; @@ -90,192 +92,215 @@ describe('Multi-Object Versioning Delete Success', function success() { objectsRes = results; }); - afterEach(done => { removeAllVersions({ Bucket: bucketName }, err => { if (err) { return done(err); } - return s3.send(new DeleteBucketCommand({ Bucket: bucketName })) - .then(() => done()).catch(done); + return s3 + .send(new DeleteBucketCommand({ Bucket: bucketName })) + .then(() => done()) + .catch(done); }); }); it('should batch delete 1000 objects quietly', async () => { - const objects = objectsRes.slice(0, 1000).map(obj => - ({ Key: obj.Key, VersionId: obj.VersionId })); - const res = await s3.send(new DeleteObjectsCommand({ - Bucket: bucketName, - Delete: { - Objects: objects, - Quiet: true, - }, - })); + const objects = objectsRes.slice(0, 1000).map(obj => ({ Key: obj.Key, VersionId: obj.VersionId })); + const res = await s3.send( + new DeleteObjectsCommand({ + Bucket: bucketName, + Delete: { + Objects: objects, + Quiet: true, + }, + }), + ); assert.strictEqual(res.Deleted, undefined); assert.strictEqual(res.Errors, undefined); }); it('should batch delete 1000 objects', async () => { - const objects = objectsRes.slice(0, 1000).map(obj => - ({ Key: obj.Key, VersionId: obj.VersionId })); - const res = await s3.send(new DeleteObjectsCommand({ - Bucket: bucketName, - Delete: { - Objects: objects, - Quiet: false, - }, - })); + const objects = objectsRes.slice(0, 1000).map(obj => ({ Key: obj.Key, VersionId: obj.VersionId })); + const res = await s3.send( + new DeleteObjectsCommand({ + Bucket: bucketName, + Delete: { + Objects: objects, + Quiet: false, + }, + }), + ); assert.strictEqual(res.Deleted.length, 1000); // order of returned objects not sorted - assert.deepStrictEqual(sortList(res.Deleted), - sortList(objects)); + assert.deepStrictEqual(sortList(res.Deleted), sortList(objects)); assert.strictEqual(res.Errors, undefined); }); - it('should return NoSuchVersion in errors if one versionId is ' + - 'invalid', async () => { - const objects = objectsRes.slice(0, 1000).map(obj => - ({ Key: obj.Key, VersionId: obj.VersionId })); + it('should return NoSuchVersion in errors if one versionId is ' + 'invalid', async () => { + const objects = objectsRes.slice(0, 1000).map(obj => ({ Key: obj.Key, VersionId: obj.VersionId })); objects[0].VersionId = 'invalid-version-id'; - - const res = await s3.send(new DeleteObjectsCommand({ - Bucket: bucketName, - Delete: { - Objects: objects, - }, - })); + + const res = await s3.send( + new DeleteObjectsCommand({ + Bucket: bucketName, + Delete: { + Objects: objects, + }, + }), + ); assert.strictEqual(res.Deleted.length, 999); assert.strictEqual(res.Errors.length, 1); assert.strictEqual(res.Errors[0].Code, 'NoSuchVersion'); }); - it('should not send back any error if a versionId does not exist ' + - 'and should not create a new delete marker', async () => { - const objects = objectsRes.slice(0, 1000).map(obj => - ({ Key: obj.Key, VersionId: obj.VersionId })); - objects[0].VersionId = nonExistingId; - const res = await s3.send(new DeleteObjectsCommand({ - Bucket: bucketName, - Delete: { - Objects: objects, - }, - })); - assert.strictEqual(res.Deleted.length, 1000); - assert.strictEqual(res.Errors, undefined); - const foundVersionId = res.Deleted.find(entry => - entry.VersionId === nonExistingId); - assert(foundVersionId); - assert.strictEqual(foundVersionId.DeleteMarker, undefined); - }); + it( + 'should not send back any error if a versionId does not exist ' + + 'and should not create a new delete marker', + async () => { + const objects = objectsRes.slice(0, 1000).map(obj => ({ Key: obj.Key, VersionId: obj.VersionId })); + objects[0].VersionId = nonExistingId; + const res = await s3.send( + new DeleteObjectsCommand({ + Bucket: bucketName, + Delete: { + Objects: objects, + }, + }), + ); + assert.strictEqual(res.Deleted.length, 1000); + assert.strictEqual(res.Errors, undefined); + const foundVersionId = res.Deleted.find(entry => entry.VersionId === nonExistingId); + assert(foundVersionId); + assert.strictEqual(foundVersionId.DeleteMarker, undefined); + }, + ); it('should not crash when deleting a null versionId that does not exist', async () => { const objects = [{ Key: objectsRes[0].Key, VersionId: 'null' }]; - const res = await s3.send(new DeleteObjectsCommand({ - Bucket: bucketName, - Delete: { - Objects: objects, - }, - })); + const res = await s3.send( + new DeleteObjectsCommand({ + Bucket: bucketName, + Delete: { + Objects: objects, + }, + }), + ); assert.deepStrictEqual(res.Deleted, [{ Key: objectsRes[0].Key, VersionId: 'null' }]); assert.strictEqual(res.Errors, undefined); }); }); }); -describe('Multi-Object Versioning Delete - deleting delete marker', -() => { +describe('Multi-Object Versioning Delete - deleting delete marker', () => { withV4(sigCfg => { const bucketUtil = new BucketUtility('default', sigCfg); const s3 = bucketUtil.s3; beforeEach(async () => { await s3.send(new CreateBucketCommand({ Bucket: bucketName })); - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: { - Status: 'Enabled', - }, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: { + Status: 'Enabled', + }, + }), + ); }); - afterEach(done => { + afterEach(done => { removeAllVersions({ Bucket: bucketName }, err => { if (err) { return done(err); } - return s3.send(new DeleteBucketCommand({ Bucket: bucketName })) - .then(() => done()).catch(done); + return s3 + .send(new DeleteBucketCommand({ Bucket: bucketName })) + .then(() => done()) + .catch(done); }); }); - it('should send back VersionId and DeleteMarkerVersionId both equal ' + - 'to deleteVersionId', async () => { + it('should send back VersionId and DeleteMarkerVersionId both equal ' + 'to deleteVersionId', async () => { await s3.send(new PutObjectCommand({ Bucket: bucketName, Key: key })); - - const deleteRes = await s3.send(new DeleteObjectCommand({ - Bucket: bucketName, - Key: key - })); + + const deleteRes = await s3.send( + new DeleteObjectCommand({ + Bucket: bucketName, + Key: key, + }), + ); const deleteVersionId = deleteRes.VersionId; - const deleteObjectsRes = await s3.send(new DeleteObjectsCommand({ - Bucket: bucketName, - Delete: { - Objects: [ - { - Key: key, - VersionId: deleteVersionId, - }, - ], - } - })); + const deleteObjectsRes = await s3.send( + new DeleteObjectsCommand({ + Bucket: bucketName, + Delete: { + Objects: [ + { + Key: key, + VersionId: deleteVersionId, + }, + ], + }, + }), + ); assert.strictEqual(deleteObjectsRes.Deleted[0].DeleteMarker, true); assert.strictEqual(deleteObjectsRes.Deleted[0].VersionId, deleteVersionId); assert.strictEqual(deleteObjectsRes.Deleted[0].DeleteMarkerVersionId, deleteVersionId); }); - it('should send back a DeleteMarkerVersionId matching the versionId ' + - 'stored for the object if trying to delete an object that does not exist', async () => { - const deleteRes = await s3.send(new DeleteObjectsCommand({ - Bucket: bucketName, - Delete: { - Objects: [ - { - Key: key, + it( + 'should send back a DeleteMarkerVersionId matching the versionId ' + + 'stored for the object if trying to delete an object that does not exist', + async () => { + const deleteRes = await s3.send( + new DeleteObjectsCommand({ + Bucket: bucketName, + Delete: { + Objects: [ + { + Key: key, + }, + ], }, - ], - } - })); + }), + ); - const versionIdFromDeleteObjects = deleteRes.Deleted[0].DeleteMarkerVersionId; - assert.strictEqual(deleteRes.Deleted[0].DeleteMarker, true); + const versionIdFromDeleteObjects = deleteRes.Deleted[0].DeleteMarkerVersionId; + assert.strictEqual(deleteRes.Deleted[0].DeleteMarker, true); - const listRes = await s3.send(new ListObjectVersionsCommand({ Bucket: bucketName })); - const versionIdFromListObjectVersions = listRes.DeleteMarkers[0].VersionId; - assert.strictEqual(versionIdFromDeleteObjects, versionIdFromListObjectVersions); - }); + const listRes = await s3.send(new ListObjectVersionsCommand({ Bucket: bucketName })); + const versionIdFromListObjectVersions = listRes.DeleteMarkers[0].VersionId; + assert.strictEqual(versionIdFromDeleteObjects, versionIdFromListObjectVersions); + }, + ); - it('should send back a DeleteMarkerVersionId matching the versionId ' + - 'stored for the object if object exists but no version was specified', async () => { - const putRes = await s3.send(new PutObjectCommand({ Bucket: bucketName, Key: key })); - const versionId = putRes.VersionId; - const deleteRes = await s3.send(new DeleteObjectsCommand({ - Bucket: bucketName, - Delete: { - Objects: [ - { - Key: key, + it( + 'should send back a DeleteMarkerVersionId matching the versionId ' + + 'stored for the object if object exists but no version was specified', + async () => { + const putRes = await s3.send(new PutObjectCommand({ Bucket: bucketName, Key: key })); + const versionId = putRes.VersionId; + const deleteRes = await s3.send( + new DeleteObjectsCommand({ + Bucket: bucketName, + Delete: { + Objects: [ + { + Key: key, + }, + ], }, - ], - } - })); + }), + ); - assert.strictEqual(deleteRes.Deleted[0].DeleteMarker, true); - const deleteVersionId = deleteRes.Deleted[0].DeleteMarkerVersionId; - assert.notEqual(deleteVersionId, versionId); + assert.strictEqual(deleteRes.Deleted[0].DeleteMarker, true); + const deleteVersionId = deleteRes.Deleted[0].DeleteMarkerVersionId; + assert.notEqual(deleteVersionId, versionId); - const listRes = await s3.send(new ListObjectVersionsCommand({ Bucket: bucketName })); - assert.strictEqual(deleteVersionId, listRes.DeleteMarkers[0].VersionId); - assert.strictEqual(versionId, listRes.Versions[0].VersionId); - }); + const listRes = await s3.send(new ListObjectVersionsCommand({ Bucket: bucketName })); + assert.strictEqual(deleteVersionId, listRes.DeleteMarkers[0].VersionId); + assert.strictEqual(versionId, listRes.Versions[0].VersionId); + }, + ); }); }); diff --git a/tests/functional/aws-node-sdk/test/versioning/objectACL.js b/tests/functional/aws-node-sdk/test/versioning/objectACL.js index b03e5a3ccf..2f65a19cb1 100644 --- a/tests/functional/aws-node-sdk/test/versioning/objectACL.js +++ b/tests/functional/aws-node-sdk/test/versioning/objectACL.js @@ -26,9 +26,9 @@ const key = '/'; const invalidId = 'invalidIdWithMoreThan40BytesAndThatIsNotLongEnoughYet'; // formats differ for AWS and S3, use respective sample ids to obtain // correct error response in tests -const nonExistingId = process.env.AWS_ON_AIR ? - 'MhhyTHhmZ4cxSi4Y9SMe5P7UJAz7HLJ9' : - '3939393939393939393936493939393939393939756e6437'; +const nonExistingId = process.env.AWS_ON_AIR + ? 'MhhyTHhmZ4cxSi4Y9SMe5P7UJAz7HLJ9' + : '3939393939393939393936493939393939393939756e6437'; class _Utils { constructor(s3) { @@ -44,25 +44,30 @@ class _Utils { async _wrapDataObject(method, params) { const Command = method === 'getObjectAcl' ? GetObjectAclCommand : PutObjectAclCommand; const data = await this.s3.send(new Command(params)); - + let versionId = params.VersionId; - + if (!versionId) { // For non-version-specific ACL operations, we need to determine the latest version try { - const headResult = await this.s3.send(new HeadObjectCommand({ - Bucket: params.Bucket, - Key: params.Key - })); + const headResult = await this.s3.send( + new HeadObjectCommand({ + Bucket: params.Bucket, + Key: params.Key, + }), + ); versionId = headResult.VersionId; } catch { versionId = undefined; // Fallback } } - - const dataObj = Object.assign({ - VersionId: versionId, - }, data); + + const dataObj = Object.assign( + { + VersionId: versionId, + }, + data, + ); return dataObj; } @@ -83,18 +88,20 @@ class _Utils { if (versionId) { params.VersionId = versionId; } - + try { const data = await this.putObjectAcl(params); if (expected.error) { // Should not reach here if error was expected assert.fail('Expected error but operation succeeded'); } - _Utils.assertNoError(null, - `putting object acl with version id: ${versionId}`); - assert.strictEqual(data.VersionId, expected.versionId, + _Utils.assertNoError(null, `putting object acl with version id: ${versionId}`); + assert.strictEqual( + data.VersionId, + expected.versionId, `expected version id '${expected.versionId}' in ` + - `putacl res headers, got '${data.VersionId}' instead`); + `putacl res headers, got '${data.VersionId}' instead`, + ); } catch (err) { if (expected.error) { assert.strictEqual(expected.error.code, err.Code); @@ -103,19 +110,20 @@ class _Utils { throw err; } } - + delete params.ACL; - + try { const data = await this.getObjectAcl(params); if (expected.error) { assert.fail('Expected error but operation succeeded'); } - _Utils.assertNoError(null, - `getting object acl with version id: ${versionId}`); - assert.strictEqual(data.VersionId, expected.versionId, - `expected version id '${expected.versionId}' in ` + - `getacl res headers, got '${data.VersionId}'`); + _Utils.assertNoError(null, `getting object acl with version id: ${versionId}`); + assert.strictEqual( + data.VersionId, + expected.versionId, + `expected version id '${expected.versionId}' in ` + `getacl res headers, got '${data.VersionId}'`, + ); assert.strictEqual(data.Grants.length, 2); } catch (err) { if (expected.error) { @@ -131,111 +139,124 @@ class _Utils { function _testBehaviorVersioningEnabledOrSuspended(utils, versionIds) { const s3 = utils.s3; - it('should return 405 MethodNotAllowed putting acl without ' + - 'version id if latest version is a delete marker', async () => { - const aclParams = { - Bucket: bucket, - Key: key, - ACL: 'public-read-write', - }; - const data = await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: key })); - assert.strictEqual(data.DeleteMarker, true); - assert(data.VersionId); - - try { - await utils.putObjectAcl(aclParams); - assert.fail('Expected error but operation succeeded'); - } catch (err) { - assert(err); - assert.strictEqual(err.Code, 'MethodNotAllowed'); - assert.strictEqual(err.$metadata.httpStatusCode, 405); - } - }); - - it('should return 405 MethodNotAllowed putting acl with ' + - 'version id if version specified is a delete marker', async () => { - const aclParams = { - Bucket: bucket, - Key: key, - ACL: 'public-read-write', - }; - const data = await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: key })); - assert.strictEqual(data.DeleteMarker, true); - assert(data.VersionId); - aclParams.VersionId = data.VersionId; - - try { - await utils.putObjectAcl(aclParams); - assert.fail('Expected error but operation succeeded'); - } catch (err) { - assert(err); - assert.strictEqual(err.Code, 'MethodNotAllowed'); - assert.strictEqual(err.$metadata.httpStatusCode, 405); - } - }); + it( + 'should return 405 MethodNotAllowed putting acl without ' + 'version id if latest version is a delete marker', + async () => { + const aclParams = { + Bucket: bucket, + Key: key, + ACL: 'public-read-write', + }; + const data = await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: key })); + assert.strictEqual(data.DeleteMarker, true); + assert(data.VersionId); - it('should return 404 NoSuchKey getting acl without ' + - 'version id if latest version is a delete marker', async () => { - const aclParams = { - Bucket: bucket, - Key: key, - }; - const data = await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: key })); - assert.strictEqual(data.DeleteMarker, true); - assert(data.VersionId); - - try { - await utils.getObjectAcl(aclParams); - assert.fail('Expected error but operation succeeded'); - } catch (err) { - assert(err); - assert.strictEqual(err.Code, 'NoSuchKey'); - assert.strictEqual(err.$metadata.httpStatusCode, 404); - } - }); + try { + await utils.putObjectAcl(aclParams); + assert.fail('Expected error but operation succeeded'); + } catch (err) { + assert(err); + assert.strictEqual(err.Code, 'MethodNotAllowed'); + assert.strictEqual(err.$metadata.httpStatusCode, 405); + } + }, + ); + + it( + 'should return 405 MethodNotAllowed putting acl with ' + 'version id if version specified is a delete marker', + async () => { + const aclParams = { + Bucket: bucket, + Key: key, + ACL: 'public-read-write', + }; + const data = await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: key })); + assert.strictEqual(data.DeleteMarker, true); + assert(data.VersionId); + aclParams.VersionId = data.VersionId; - it('should return 405 MethodNotAllowed getting acl with ' + - 'version id if version specified is a delete marker', async () => { - const latestVersion = versionIds[versionIds.length - 1]; - const aclParams = { - Bucket: bucket, - Key: key, - VersionId: latestVersion, - }; - const data = await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: key })); - assert.strictEqual(data.DeleteMarker, true); - assert(data.VersionId); - aclParams.VersionId = data.VersionId; - - try { - await utils.getObjectAcl(aclParams); - assert.fail('Expected error but operation succeeded'); - } catch (err) { - assert(err); - assert.strictEqual(err.Code, 'MethodNotAllowed'); - assert.strictEqual(err.$metadata.httpStatusCode, 405); - } - }); + try { + await utils.putObjectAcl(aclParams); + assert.fail('Expected error but operation succeeded'); + } catch (err) { + assert(err); + assert.strictEqual(err.Code, 'MethodNotAllowed'); + assert.strictEqual(err.$metadata.httpStatusCode, 405); + } + }, + ); + + it( + 'should return 404 NoSuchKey getting acl without ' + 'version id if latest version is a delete marker', + async () => { + const aclParams = { + Bucket: bucket, + Key: key, + }; + const data = await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: key })); + assert.strictEqual(data.DeleteMarker, true); + assert(data.VersionId); - it('non-version specific put and get ACL should target latest ' + - 'version AND return version ID in response headers', async () => { - const latestVersion = versionIds[versionIds.length - 1]; - const expectedRes = { versionId: latestVersion }; - await utils.putAndGetAcl('public-read', undefined, expectedRes); - }); + try { + await utils.getObjectAcl(aclParams); + assert.fail('Expected error but operation succeeded'); + } catch (err) { + assert(err); + assert.strictEqual(err.Code, 'NoSuchKey'); + assert.strictEqual(err.$metadata.httpStatusCode, 404); + } + }, + ); + + it( + 'should return 405 MethodNotAllowed getting acl with ' + 'version id if version specified is a delete marker', + async () => { + const latestVersion = versionIds[versionIds.length - 1]; + const aclParams = { + Bucket: bucket, + Key: key, + VersionId: latestVersion, + }; + const data = await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: key })); + assert.strictEqual(data.DeleteMarker, true); + assert(data.VersionId); + aclParams.VersionId = data.VersionId; - it('version specific put and get ACL should return version ID ' + - 'in response headers', async () => { + try { + await utils.getObjectAcl(aclParams); + assert.fail('Expected error but operation succeeded'); + } catch (err) { + assert(err); + assert.strictEqual(err.Code, 'MethodNotAllowed'); + assert.strictEqual(err.$metadata.httpStatusCode, 405); + } + }, + ); + + it( + 'non-version specific put and get ACL should target latest ' + + 'version AND return version ID in response headers', + async () => { + const latestVersion = versionIds[versionIds.length - 1]; + const expectedRes = { versionId: latestVersion }; + await utils.putAndGetAcl('public-read', undefined, expectedRes); + }, + ); + + it('version specific put and get ACL should return version ID ' + 'in response headers', async () => { const firstVersion = versionIds[0]; const expectedRes = { versionId: firstVersion }; await utils.putAndGetAcl('public-read', firstVersion, expectedRes); }); - it('version specific put and get ACL (version id = "null") ' + - 'should return version ID ("null") in response headers', async () => { - const expectedRes = { versionId: 'null' }; - await utils.putAndGetAcl('public-read', 'null', expectedRes); - }); + it( + 'version specific put and get ACL (version id = "null") ' + + 'should return version ID ("null") in response headers', + async () => { + const expectedRes = { versionId: 'null' }; + await utils.putAndGetAcl('public-read', 'null', expectedRes); + }, + ); } describe('versioned put and get object acl ::', () => { @@ -254,8 +275,10 @@ describe('versioned put and get object acl ::', () => { if (err) { return done(err); } - return s3.send(new DeleteBucketCommand({ Bucket: bucket })) - .then(() => done()).catch(done); + return s3 + .send(new DeleteBucketCommand({ Bucket: bucket })) + .then(() => done()) + .catch(done); }); }); @@ -264,42 +287,45 @@ describe('versioned put and get object acl ::', () => { await s3.send(new PutObjectCommand({ Bucket: bucket, Key: key })); }); - it('should not return version id for non-version specific ' + - 'put and get ACL', async () => { + it('should not return version id for non-version specific ' + 'put and get ACL', async () => { const expectedRes = { versionId: undefined }; await utils.putAndGetAcl('public-read', undefined, expectedRes); }); - it('should not return version id for version specific ' + - 'put and get ACL (version id = "null")', async () => { - const expectedRes = { versionId: 'null' }; - await utils.putAndGetAcl('public-read', 'null', expectedRes); - }); - - it('should return NoSuchVersion if attempting to put or get acl ' + - 'for non-existing version', async () => { - const error = { code: 'NoSuchVersion', statusCode: 404 }; - await utils.putAndGetAcl('private', nonExistingId, { error }); - }); - - it('should return InvalidArgument if attempting to put/get acl ' + - 'for invalid hex string', async () => { + it( + 'should not return version id for version specific ' + 'put and get ACL (version id = "null")', + async () => { + const expectedRes = { versionId: 'null' }; + await utils.putAndGetAcl('public-read', 'null', expectedRes); + }, + ); + + it( + 'should return NoSuchVersion if attempting to put or get acl ' + 'for non-existing version', + async () => { + const error = { code: 'NoSuchVersion', statusCode: 404 }; + await utils.putAndGetAcl('private', nonExistingId, { error }); + }, + ); + + it('should return InvalidArgument if attempting to put/get acl ' + 'for invalid hex string', async () => { const error = { code: 'InvalidArgument', statusCode: 400 }; await utils.putAndGetAcl('private', invalidId, { error }); }); }); - describe('on a version-enabled bucket with non-versioned object :: ', - () => { + describe('on a version-enabled bucket with non-versioned object :: ', () => { const versionIds = []; beforeEach(async () => { const params = { Bucket: bucket, Key: key }; await s3.send(new PutObjectCommand(params)); - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningEnabled, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningEnabled, + }), + ); }); afterEach(() => { @@ -308,11 +334,14 @@ describe('versioned put and get object acl ::', () => { }); describe('before putting new versions :: ', () => { - it('non-version specific put and get ACL should now ' + - 'return version ID ("null") in response headers', async () => { - const expectedRes = { versionId: 'null' }; - await utils.putAndGetAcl('public-read', undefined, expectedRes); - }); + it( + 'non-version specific put and get ACL should now ' + + 'return version ID ("null") in response headers', + async () => { + const expectedRes = { versionId: 'null' }; + await utils.putAndGetAcl('public-read', undefined, expectedRes); + }, + ); }); describe('after putting new versions :: ', () => { @@ -329,39 +358,43 @@ describe('versioned put and get object acl ::', () => { }); }); - describe('on a version-enabled bucket - version non-specified :: ', - () => { + describe('on a version-enabled bucket - version non-specified :: ', () => { let versionId; beforeEach(async () => { const params = { Bucket: bucket, Key: key }; - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningEnabled, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningEnabled, + }), + ); const data = await s3.send(new PutObjectCommand(params)); versionId = data.VersionId; }); - it('should not create version putting ACL on a' + - 'version-enabled bucket where no version id is specified', - async () => { - const params = { Bucket: bucket, Key: key, ACL: 'public-read' }; - await utils.putObjectAcl(params); - await checkOneVersion(s3, bucket, versionId); - }); + it( + 'should not create version putting ACL on a' + + 'version-enabled bucket where no version id is specified', + async () => { + const params = { Bucket: bucket, Key: key, ACL: 'public-read' }; + await utils.putObjectAcl(params); + await checkOneVersion(s3, bucket, versionId); + }, + ); }); - describe('on version-suspended bucket with non-versioned object :: ', - () => { + describe('on version-suspended bucket with non-versioned object :: ', () => { const versionIds = []; beforeEach(async () => { const params = { Bucket: bucket, Key: key }; await s3.send(new PutObjectCommand(params)); - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningSuspended, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningSuspended, + }), + ); }); afterEach(() => { @@ -370,31 +403,38 @@ describe('versioned put and get object acl ::', () => { }); describe('before putting new versions :: ', () => { - it('non-version specific put and get ACL should still ' + - 'return version ID ("null") in response headers', async () => { - const expectedRes = { versionId: 'null' }; - await utils.putAndGetAcl('public-read', undefined, expectedRes); - }); + it( + 'non-version specific put and get ACL should still ' + + 'return version ID ("null") in response headers', + async () => { + const expectedRes = { versionId: 'null' }; + await utils.putAndGetAcl('public-read', undefined, expectedRes); + }, + ); }); describe('after putting new versions :: ', () => { beforeEach(async () => { const params = { Bucket: bucket, Key: key }; - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningEnabled, - })); - + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningEnabled, + }), + ); + for (let i = 0; i < counter; i++) { const data = await s3.send(new PutObjectCommand(params)); _Utils.assertNoError(null, `putting version #${i}`); versionIds.push(data.VersionId); } - - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningSuspended, - })); + + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningSuspended, + }), + ); }); _testBehaviorVersioningEnabledOrSuspended(utils, versionIds); diff --git a/tests/functional/aws-node-sdk/test/versioning/objectCopy.js b/tests/functional/aws-node-sdk/test/versioning/objectCopy.js index 7158aa0a42..e1b8aedf57 100644 --- a/tests/functional/aws-node-sdk/test/versioning/objectCopy.js +++ b/tests/functional/aws-node-sdk/test/versioning/objectCopy.js @@ -22,7 +22,6 @@ const customS3Request = require('../../lib/utility/customS3Request'); const removeAllVersionsPromise = promisify(removeAllVersions); - const { taggingTests } = require('../../lib/utility/tagging'); const constants = require('../../../../../constants'); @@ -64,8 +63,7 @@ const otherAccountBucketUtility = new BucketUtility('lisa', {}); const otherAccountS3 = otherAccountBucketUtility.s3; function checkNoError(err) { - assert.equal(err, null, - `Expected success, got error ${JSON.stringify(err)}`); + assert.equal(err, null, `Expected success, got error ${JSON.stringify(err)}`); } function checkError(err, code) { @@ -102,52 +100,59 @@ describe('Object Version Copy', () => { beforeEach(async () => { await bucketUtil.createOne(sourceBucketName); await bucketUtil.createOne(destBucketName); - await s3.send(new PutBucketVersioningCommand({ - Bucket: sourceBucketName, - VersioningConfiguration: { Status: 'Enabled' }, - })); - const putRes = await s3.send(new PutObjectCommand({ - Bucket: sourceBucketName, - Key: sourceObjName, - Body: content, - Metadata: originalMetadata, - CacheControl: originalCacheControl, - ContentDisposition: originalContentDisposition, - ContentEncoding: originalContentEncoding, - Expires: originalExpires, - Tagging: originalTagging, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: sourceBucketName, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ); + const putRes = await s3.send( + new PutObjectCommand({ + Bucket: sourceBucketName, + Key: sourceObjName, + Body: content, + Metadata: originalMetadata, + CacheControl: originalCacheControl, + ContentDisposition: originalContentDisposition, + ContentEncoding: originalContentEncoding, + Expires: originalExpires, + Tagging: originalTagging, + }), + ); etag = putRes.ETag; versionId = putRes.VersionId; copySource = `${sourceBucketName}/${sourceObjName}?versionId=${versionId}`; etagTrim = etag.substring(1, etag.length - 1); copySourceVersionId = putRes.VersionId; - const headRes = await s3.send(new HeadObjectCommand({ - Bucket: sourceBucketName, - Key: sourceObjName, - })); + const headRes = await s3.send( + new HeadObjectCommand({ + Bucket: sourceBucketName, + Key: sourceObjName, + }), + ); lastModified = headRes.LastModified; - await s3.send(new PutObjectCommand({ - Bucket: sourceBucketName, - Key: sourceObjName, - Body: secondContent, - })); + await s3.send( + new PutObjectCommand({ + Bucket: sourceBucketName, + Key: sourceObjName, + Body: secondContent, + }), + ); }); afterEach(async () => { - await Promise.all([ - emptyAndDeleteBucket(sourceBucketName), - emptyAndDeleteBucket(destBucketName), - ]); + await Promise.all([emptyAndDeleteBucket(sourceBucketName), emptyAndDeleteBucket(destBucketName)]); }); async function requestCopy(fields) { - return s3.send(new CopyObjectCommand({ - Bucket: destBucketName, - Key: destObjName, - CopySource: copySource, - ...fields, - })); + return s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: copySource, + ...fields, + }), + ); } async function successCopyCheck(error, response, copyVersionMetadata, destBucketName, destObjName) { @@ -157,8 +162,7 @@ describe('Object Version Copy', () => { const destinationVersionId = response.VersionId; assert.strictEqual(response.CopyObjectResult.ETag, etag); const copyLastModified = new Date(response.CopyObjectResult.LastModified).toGMTString(); - const res = await s3.send(new GetObjectCommand({ Bucket: destBucketName, - Key: destObjName })); + const res = await s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })); assert.strictEqual(res.VersionId, destinationVersionId); const responseBody = await res.Body.transformToString(); assert.strictEqual(responseBody, content); @@ -172,30 +176,51 @@ describe('Object Version Copy', () => { assert.strictEqual(data.TagSet[0].Value, value); } - it('should copy an object from a source bucket to a different '+ - 'destination bucket and copy the tag set if no tagging directive '+ - 'header provided', async () => { - await s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: copySource })); - await checkSuccessTagging(originalTagKey, originalTagValue); - }); - - it('should copy an object from a source bucket to a different ' + - 'destination bucket and copy the tag set if COPY tagging ' + - 'directive header provided', async () => { - await s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: copySource, - TaggingDirective: 'COPY' })); - await checkSuccessTagging(originalTagKey, originalTagValue); - }); - - it('should copy an object from a source to the same destination '+ - 'updating tag if REPLACE tagging directive header provided', async () => { - await s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: copySource, - TaggingDirective: 'REPLACE', Tagging: newTagging })); - await checkSuccessTagging(newTagKey, newTagValue); - }); + it( + 'should copy an object from a source bucket to a different ' + + 'destination bucket and copy the tag set if no tagging directive ' + + 'header provided', + async () => { + await s3.send( + new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, CopySource: copySource }), + ); + await checkSuccessTagging(originalTagKey, originalTagValue); + }, + ); + + it( + 'should copy an object from a source bucket to a different ' + + 'destination bucket and copy the tag set if COPY tagging ' + + 'directive header provided', + async () => { + await s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: copySource, + TaggingDirective: 'COPY', + }), + ); + await checkSuccessTagging(originalTagKey, originalTagValue); + }, + ); + + it( + 'should copy an object from a source to the same destination ' + + 'updating tag if REPLACE tagging directive header provided', + async () => { + await s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: copySource, + TaggingDirective: 'REPLACE', + Tagging: newTagging, + }), + ); + await checkSuccessTagging(newTagKey, newTagValue); + }, + ); describe('Copy object with versioning updating tag set', () => { taggingTests.forEach(taggingTest => { @@ -203,9 +228,13 @@ describe('Object Version Copy', () => { const key = encodeURIComponent(taggingTest.tag.key); const value = encodeURIComponent(taggingTest.tag.value); const tagging = `${key}=${value}`; - const params = { Bucket: destBucketName, Key: destObjName, CopySource: copySource, + const params = { + Bucket: destBucketName, + Key: destObjName, + CopySource: copySource, TaggingDirective: 'REPLACE', - Tagging: tagging }; + Tagging: tagging, + }; try { await s3.send(new CopyObjectCommand(params)); await checkSuccessTagging(taggingTest.tag.key, taggingTest.tag.value); @@ -232,8 +261,7 @@ describe('Object Version Copy', () => { } }); - it('should return InvalidArgument for a request with empty string '+ - 'versionId query', async () => { + it('should return InvalidArgument for a request with empty string ' + 'versionId query', async () => { const params = { Bucket: destBucketName, Key: destObjName, CopySource: copySource }; const query = { versionId: '' }; try { @@ -245,137 +273,193 @@ describe('Object Version Copy', () => { } }); - it('should copy a version from a source bucket to a different' + - 'destination bucket and copy the metadata if no metadata directive' + - 'header provided', async () => { - const res = await s3.send(new CopyObjectCommand({ Bucket: destBucketName, - Key: destObjName, - CopySource: copySource })); - await successCopyCheck(null, res, originalMetadata, destBucketName, destObjName); - }); - - it('should also copy additional headers (CacheControl, ' + - 'ContentDisposition, ContentEncoding, Expires) when copying an ' + - 'object from a source bucket to a different destination bucket', async () => { - await s3.send(new CopyObjectCommand({ Bucket: destBucketName, - Key: destObjName, - CopySource: copySource })); - const res = await s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })); - assert.strictEqual(res.CacheControl, originalCacheControl); - assert.strictEqual(res.ContentDisposition, originalContentDisposition); - assert.strictEqual(res.ContentEncoding, 'base64,'); - assert.strictEqual(res.Expires.toGMTString(), originalExpires.toGMTString()); - }); - - it('should copy an object from a source bucket to a different '+ - 'key in the same bucket', async () => { - const res = await s3.send(new CopyObjectCommand({ Bucket: sourceBucketName, - Key: destObjName, - CopySource: copySource })); - await successCopyCheck(null, res, originalMetadata, - sourceBucketName, destObjName); - }); - - it('should copy an object from a source to the same destination ' + - '(update metadata)', async () => { - const res = await s3.send(new CopyObjectCommand({ Bucket: sourceBucketName, Key: sourceObjName, - CopySource: copySource, - MetadataDirective: 'REPLACE', - Metadata: newMetadata })); + it( + 'should copy a version from a source bucket to a different' + + 'destination bucket and copy the metadata if no metadata directive' + + 'header provided', + async () => { + const res = await s3.send( + new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, CopySource: copySource }), + ); + await successCopyCheck(null, res, originalMetadata, destBucketName, destObjName); + }, + ); + + it( + 'should also copy additional headers (CacheControl, ' + + 'ContentDisposition, ContentEncoding, Expires) when copying an ' + + 'object from a source bucket to a different destination bucket', + async () => { + await s3.send( + new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, CopySource: copySource }), + ); + const res = await s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })); + assert.strictEqual(res.CacheControl, originalCacheControl); + assert.strictEqual(res.ContentDisposition, originalContentDisposition); + assert.strictEqual(res.ContentEncoding, 'base64,'); + assert.strictEqual(res.Expires.toGMTString(), originalExpires.toGMTString()); + }, + ); + + it('should copy an object from a source bucket to a different ' + 'key in the same bucket', async () => { + const res = await s3.send( + new CopyObjectCommand({ Bucket: sourceBucketName, Key: destObjName, CopySource: copySource }), + ); + await successCopyCheck(null, res, originalMetadata, sourceBucketName, destObjName); + }); + + it('should copy an object from a source to the same destination ' + '(update metadata)', async () => { + const res = await s3.send( + new CopyObjectCommand({ + Bucket: sourceBucketName, + Key: sourceObjName, + CopySource: copySource, + MetadataDirective: 'REPLACE', + Metadata: newMetadata, + }), + ); await successCopyCheck(null, res, newMetadata, sourceBucketName, sourceObjName); }); - it('should copy an object and replace the metadata if replace ' + - 'included as metadata directive header', async () => { - const res = await s3.send(new CopyObjectCommand({ Bucket: destBucketName, - Key: destObjName, - CopySource: copySource, - MetadataDirective: 'REPLACE', - Metadata: newMetadata })); - await successCopyCheck(null, res, newMetadata, destBucketName, destObjName); - }); - - it('should copy an object and replace ContentType if replace ' + - 'included as a metadata directive header, and new ContentType is ' + - 'provided', async () => { - await s3.send(new CopyObjectCommand({ Bucket: destBucketName, - Key: destObjName, - CopySource: copySource, - MetadataDirective: 'REPLACE', - ContentType: 'image' })); - const res = await s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })); - assert.strictEqual(res.ContentType, 'image'); - }); - - it('should copy an object and keep ContentType if replace ' + - 'included as a metadata directive header, but no new ContentType ' + - 'is provided', async () => { - await s3.send(new CopyObjectCommand({ Bucket: destBucketName, - Key: destObjName, - CopySource: copySource, - MetadataDirective: 'REPLACE' })); - const res = await s3.send(new GetObjectCommand({ Bucket: destBucketName, - Key: destObjName })); - assert.strictEqual(res.ContentType, 'application/octet-stream'); - }); - - it('should also replace additional headers if replace ' + - 'included as metadata directive header and new headers are ' + - 'specified', async () => { - await s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: copySource, - MetadataDirective: 'REPLACE', - CacheControl: newCacheControl, - ContentDisposition: newContentDisposition, - ContentEncoding: newContentEncoding, - Expires: newExpires })); - const res = await s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })); - assert.strictEqual(res.CacheControl, newCacheControl); - assert.strictEqual(res.ContentDisposition, newContentDisposition); - assert.strictEqual(res.ContentEncoding, 'gzip,'); - assert.strictEqual(res.Expires.toGMTString(), newExpires.toGMTString()); - }); - - it('should copy an object and the metadata if copy ' + - 'included as metadata directive header (and ignore any new ' + - 'metadata sent with copy request)', async () => { - await s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: copySource, - MetadataDirective: 'COPY', - Metadata: newMetadata })); - const res = await s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })); - assert.deepStrictEqual(res.Metadata, originalMetadata); - }); - - it('should copy an object and its additional headers if copy ' + - 'included as metadata directive header (and ignore any new ' + - 'headers sent with copy request)', async () => { - await s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: copySource, - MetadataDirective: 'COPY', - Metadata: newMetadata, - CacheControl: newCacheControl, - ContentDisposition: newContentDisposition, - ContentEncoding: newContentEncoding, - Expires: newExpires })); - const res = await s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })); - assert.strictEqual(res.CacheControl, originalCacheControl); - assert.strictEqual(res.ContentDisposition, originalContentDisposition); - assert.strictEqual(res.ContentEncoding, 'base64,'); - assert.strictEqual(res.Expires.toGMTString(), originalExpires.toGMTString()); - }); + it( + 'should copy an object and replace the metadata if replace ' + 'included as metadata directive header', + async () => { + const res = await s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: copySource, + MetadataDirective: 'REPLACE', + Metadata: newMetadata, + }), + ); + await successCopyCheck(null, res, newMetadata, destBucketName, destObjName); + }, + ); + + it( + 'should copy an object and replace ContentType if replace ' + + 'included as a metadata directive header, and new ContentType is ' + + 'provided', + async () => { + await s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: copySource, + MetadataDirective: 'REPLACE', + ContentType: 'image', + }), + ); + const res = await s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })); + assert.strictEqual(res.ContentType, 'image'); + }, + ); + + it( + 'should copy an object and keep ContentType if replace ' + + 'included as a metadata directive header, but no new ContentType ' + + 'is provided', + async () => { + await s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: copySource, + MetadataDirective: 'REPLACE', + }), + ); + const res = await s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })); + assert.strictEqual(res.ContentType, 'application/octet-stream'); + }, + ); + + it( + 'should also replace additional headers if replace ' + + 'included as metadata directive header and new headers are ' + + 'specified', + async () => { + await s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: copySource, + MetadataDirective: 'REPLACE', + CacheControl: newCacheControl, + ContentDisposition: newContentDisposition, + ContentEncoding: newContentEncoding, + Expires: newExpires, + }), + ); + const res = await s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })); + assert.strictEqual(res.CacheControl, newCacheControl); + assert.strictEqual(res.ContentDisposition, newContentDisposition); + assert.strictEqual(res.ContentEncoding, 'gzip,'); + assert.strictEqual(res.Expires.toGMTString(), newExpires.toGMTString()); + }, + ); + + it( + 'should copy an object and the metadata if copy ' + + 'included as metadata directive header (and ignore any new ' + + 'metadata sent with copy request)', + async () => { + await s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: copySource, + MetadataDirective: 'COPY', + Metadata: newMetadata, + }), + ); + const res = await s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })); + assert.deepStrictEqual(res.Metadata, originalMetadata); + }, + ); + + it( + 'should copy an object and its additional headers if copy ' + + 'included as metadata directive header (and ignore any new ' + + 'headers sent with copy request)', + async () => { + await s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: copySource, + MetadataDirective: 'COPY', + Metadata: newMetadata, + CacheControl: newCacheControl, + ContentDisposition: newContentDisposition, + ContentEncoding: newContentEncoding, + Expires: newExpires, + }), + ); + const res = await s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })); + assert.strictEqual(res.CacheControl, originalCacheControl); + assert.strictEqual(res.ContentDisposition, originalContentDisposition); + assert.strictEqual(res.ContentEncoding, 'base64,'); + assert.strictEqual(res.Expires.toGMTString(), originalExpires.toGMTString()); + }, + ); it('should copy a 0 byte object to different destination', async () => { const emptyFileETag = '"d41d8cd98f00b204e9800998ecf8427e"'; - const putRes = await s3.send(new PutObjectCommand({ Bucket: sourceBucketName, Key: sourceObjName, - Body: '', - Metadata: originalMetadata })); + const putRes = await s3.send( + new PutObjectCommand({ + Bucket: sourceBucketName, + Key: sourceObjName, + Body: '', + Metadata: originalMetadata, + }), + ); copySource = `${sourceBucketName}/${sourceObjName}?versionId=${putRes.VersionId}`; - const copyRes = await s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: copySource })); + const copyRes = await s3.send( + new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, CopySource: copySource }), + ); assert.strictEqual(copyRes.CopyObjectResult.ETag, emptyFileETag); - const getRes = await s3.send(new GetObjectCommand({ Bucket: destBucketName, - Key: destObjName })); + const getRes = await s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })); assert.deepStrictEqual(getRes.Metadata, originalMetadata); assert.strictEqual(getRes.ETag, emptyFileETag); }); @@ -384,119 +468,181 @@ describe('Object Version Copy', () => { if (constants.validStorageClasses.includes('REDUCED_REDUNDANCY')) { it('should copy a 0 byte object to same destination', async () => { const emptyFileETag = '"d41d8cd98f00b204e9800998ecf8427e"'; - const putRes = await s3.send(new PutObjectCommand({ Bucket: sourceBucketName, Key: sourceObjName, - Body: '' })); + const putRes = await s3.send( + new PutObjectCommand({ Bucket: sourceBucketName, Key: sourceObjName, Body: '' }), + ); copySource = `${sourceBucketName}/${sourceObjName}?versionId=${putRes.VersionId}`; - const copyRes = await s3.send(new CopyObjectCommand({ Bucket: sourceBucketName, Key: sourceObjName, - CopySource: copySource, - StorageClass: 'REDUCED_REDUNDANCY' })); + const copyRes = await s3.send( + new CopyObjectCommand({ + Bucket: sourceBucketName, + Key: sourceObjName, + CopySource: copySource, + StorageClass: 'REDUCED_REDUNDANCY', + }), + ); assert.notEqual(copyRes.VersionId, putRes.VersionId); assert.strictEqual(copyRes.ETag, emptyFileETag); - const getRes = await s3.send(new GetObjectCommand({ Bucket: sourceBucketName, - Key: sourceObjName })); + const getRes = await s3.send(new GetObjectCommand({ Bucket: sourceBucketName, Key: sourceObjName })); assert.deepStrictEqual(getRes.Metadata, {}); - assert.strictEqual(getRes.StorageClass, - 'REDUCED_REDUNDANCY'); + assert.strictEqual(getRes.StorageClass, 'REDUCED_REDUNDANCY'); assert.strictEqual(getRes.ETag, emptyFileETag); }); - it('should copy an object to a different destination and change ' + - 'the storage class if storage class header provided', async () => { - await s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: copySource, - StorageClass: 'REDUCED_REDUNDANCY' })); - const res = await s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })); - assert.strictEqual(res.StorageClass, 'REDUCED_REDUNDANCY'); - }); - - it('should copy an object to the same destination and change the ' + - 'storage class if the storage class header provided', async () => { - await s3.send(new CopyObjectCommand({ Bucket: sourceBucketName, Key: sourceObjName, - CopySource: copySource, - StorageClass: 'REDUCED_REDUNDANCY' })); - const res = await s3.send(new GetObjectCommand({ Bucket: sourceBucketName, Key: sourceObjName })); - assert.strictEqual(res.StorageClass, 'REDUCED_REDUNDANCY'); - }); + it( + 'should copy an object to a different destination and change ' + + 'the storage class if storage class header provided', + async () => { + await s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: copySource, + StorageClass: 'REDUCED_REDUNDANCY', + }), + ); + const res = await s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })); + assert.strictEqual(res.StorageClass, 'REDUCED_REDUNDANCY'); + }, + ); + + it( + 'should copy an object to the same destination and change the ' + + 'storage class if the storage class header provided', + async () => { + await s3.send( + new CopyObjectCommand({ + Bucket: sourceBucketName, + Key: sourceObjName, + CopySource: copySource, + StorageClass: 'REDUCED_REDUNDANCY', + }), + ); + const res = await s3.send(new GetObjectCommand({ Bucket: sourceBucketName, Key: sourceObjName })); + assert.strictEqual(res.StorageClass, 'REDUCED_REDUNDANCY'); + }, + ); } - it('should copy an object to a new bucket and overwrite an already ' + - 'existing object in the destination bucket', async () => { - await s3.send(new PutObjectCommand({ Bucket: destBucketName, Key: destObjName, - Body: 'overwrite me', Metadata: originalMetadata })); - const copyRes = await s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: copySource, - MetadataDirective: 'REPLACE', - Metadata: newMetadata })); - assert.strictEqual(copyRes.CopyObjectResult.ETag, etag); - const getRes = await s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })); - assert.deepStrictEqual(getRes.Metadata, newMetadata); - assert.strictEqual(getRes.ETag, etag); - const body = await getRes.Body.transformToString(); - assert.strictEqual(body, content); - }); + it( + 'should copy an object to a new bucket and overwrite an already ' + + 'existing object in the destination bucket', + async () => { + await s3.send( + new PutObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + Body: 'overwrite me', + Metadata: originalMetadata, + }), + ); + const copyRes = await s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: copySource, + MetadataDirective: 'REPLACE', + Metadata: newMetadata, + }), + ); + assert.strictEqual(copyRes.CopyObjectResult.ETag, etag); + const getRes = await s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })); + assert.deepStrictEqual(getRes.Metadata, newMetadata); + assert.strictEqual(getRes.ETag, etag); + const body = await getRes.Body.transformToString(); + assert.strictEqual(body, content); + }, + ); // skipping test as object level encryption is not implemented yet - it.skip('should copy an object and change the server side encryption' + - 'option if server side encryption header provided', async () => { - await s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: copySource, - ServerSideEncryption: 'AES256' })); - const res = await s3.send(new GetObjectCommand({ Bucket: destBucketName, - Key: destObjName })); - assert.strictEqual(res.ServerSideEncryption, 'AES256'); - }); - - it('should return Not Implemented error for obj. encryption using '+ - 'customer-provided encryption keys', async () => { - const params = { Bucket: destBucketName, Key: 'key', - CopySource: copySource, - SSECustomerAlgorithm: 'AES256' }; - try { - await s3.send(new CopyObjectCommand(params)); - assert.fail('Expected NotImplemented error'); - } catch (err) { - assert.strictEqual(err.name, 'NotImplemented'); - } - }); + it.skip( + 'should copy an object and change the server side encryption' + + 'option if server side encryption header provided', + async () => { + await s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: copySource, + ServerSideEncryption: 'AES256', + }), + ); + const res = await s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })); + assert.strictEqual(res.ServerSideEncryption, 'AES256'); + }, + ); + + it( + 'should return Not Implemented error for obj. encryption using ' + 'customer-provided encryption keys', + async () => { + const params = { + Bucket: destBucketName, + Key: 'key', + CopySource: copySource, + SSECustomerAlgorithm: 'AES256', + }; + try { + await s3.send(new CopyObjectCommand(params)); + assert.fail('Expected NotImplemented error'); + } catch (err) { + assert.strictEqual(err.name, 'NotImplemented'); + } + }, + ); it('should copy an object and set the acl on the new object', async () => { - await s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: copySource, - ACL: 'authenticated-read' })); - const res = await s3.send(new GetObjectAclCommand({ Bucket: destBucketName, - Key: destObjName })); + await s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: copySource, + ACL: 'authenticated-read', + }), + ); + const res = await s3.send(new GetObjectAclCommand({ Bucket: destBucketName, Key: destObjName })); assert.strictEqual(res.Grants.length, 2); assert.strictEqual(res.Grants[0].Permission, 'FULL_CONTROL'); assert.strictEqual(res.Grants[1].Permission, 'READ'); - assert.strictEqual(res.Grants[1].Grantee.URI, - 'http://acs.amazonaws.com/groups/global/AuthenticatedUsers'); - }); - - it('should copy an object and default the acl on the new object ' + - 'to private even if the copied object had a ' + - 'different acl', async () => { - await s3.send(new PutObjectAclCommand({ Bucket: sourceBucketName, Key: sourceObjName, - ACL: 'authenticated-read', - VersionId: versionId })); - await s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: copySource })); - const res = await s3.send(new GetObjectAclCommand({ Bucket: destBucketName, - Key: destObjName })); - assert.strictEqual(res.Grants.length, 1); - assert.strictEqual(res.Grants[0].Permission, 'FULL_CONTROL'); - }); - - it('should copy a version to same object name to restore '+ - 'version of object', async () => { - const res = await s3.send(new CopyObjectCommand({ Bucket: sourceBucketName, Key: sourceObjName, - CopySource: copySource })); + assert.strictEqual(res.Grants[1].Grantee.URI, 'http://acs.amazonaws.com/groups/global/AuthenticatedUsers'); + }); + + it( + 'should copy an object and default the acl on the new object ' + + 'to private even if the copied object had a ' + + 'different acl', + async () => { + await s3.send( + new PutObjectAclCommand({ + Bucket: sourceBucketName, + Key: sourceObjName, + ACL: 'authenticated-read', + VersionId: versionId, + }), + ); + await s3.send( + new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, CopySource: copySource }), + ); + const res = await s3.send(new GetObjectAclCommand({ Bucket: destBucketName, Key: destObjName })); + assert.strictEqual(res.Grants.length, 1); + assert.strictEqual(res.Grants[0].Permission, 'FULL_CONTROL'); + }, + ); + + it('should copy a version to same object name to restore ' + 'version of object', async () => { + const res = await s3.send( + new CopyObjectCommand({ Bucket: sourceBucketName, Key: sourceObjName, CopySource: copySource }), + ); await successCopyCheck(null, res, originalMetadata, sourceBucketName, sourceObjName); }); it('should return an error if attempt to copy from nonexistent bucket', async () => { try { - await s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: `nobucket453234/${sourceObjName}` })); + await s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `nobucket453234/${sourceObjName}`, + }), + ); assert.fail('Expected error'); } catch (err) { checkError(err, 'NoSuchBucket'); @@ -505,9 +651,14 @@ describe('Object Version Copy', () => { it('should return an error if use invalid redirect location', async () => { try { - await s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: copySource, - WebsiteRedirectLocation: 'google.com' })); + await s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: copySource, + WebsiteRedirectLocation: 'google.com', + }), + ); assert.fail('Expected error'); } catch (err) { checkError(err, 'InvalidRedirectLocation'); @@ -516,8 +667,13 @@ describe('Object Version Copy', () => { it('should return an error if attempt to copy to nonexistent bucket', async () => { try { - await s3.send(new CopyObjectCommand({ Bucket: 'nobucket453234', Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}` })); + await s3.send( + new CopyObjectCommand({ + Bucket: 'nobucket453234', + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + }), + ); assert.fail('Expected error'); } catch (err) { checkError(err, 'NoSuchBucket'); @@ -526,8 +682,13 @@ describe('Object Version Copy', () => { it('should return an error if attempt to copy nonexistent object', async () => { try { - await s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: `${sourceBucketName}/nokey` })); + await s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/nokey`, + }), + ); assert.fail('Expected error'); } catch (err) { checkError(err, 'NoSuchKey'); @@ -535,12 +696,16 @@ describe('Object Version Copy', () => { }); it('should return NoSuchKey if attempt to copy version with delete marker', async () => { - const delRes = await s3.send(new DeleteObjectCommand({ Bucket: sourceBucketName, - Key: sourceObjName })); + const delRes = await s3.send(new DeleteObjectCommand({ Bucket: sourceBucketName, Key: sourceObjName })); assert.strictEqual(delRes.DeleteMarker, true); try { - await s3.send(new CopyObjectCommand({ Bucket: destBucketName, - Key: destObjName, CopySource: `${sourceBucketName}/${sourceObjName}` })); + await s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + }), + ); assert.fail('Expected error'); } catch (err) { checkError(err, 'NoSuchKey'); @@ -548,15 +713,17 @@ describe('Object Version Copy', () => { }); it('should return InvalidRequest if attempt to copy specific version that is a delete marker', async () => { - const delRes = await s3.send(new DeleteObjectCommand({ Bucket: sourceBucketName, - Key: sourceObjName })); + const delRes = await s3.send(new DeleteObjectCommand({ Bucket: sourceBucketName, Key: sourceObjName })); assert.strictEqual(delRes.DeleteMarker, true); const deleteMarkerId = delRes.VersionId; try { - await s3.send(new CopyObjectCommand({ Bucket: destBucketName, - Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}` + - `?versionId=${deleteMarkerId}` })); + await s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}` + `?versionId=${deleteMarkerId}`, + }), + ); assert.fail('Expected error'); } catch (err) { checkError(err, 'InvalidRequest'); @@ -565,9 +732,14 @@ describe('Object Version Copy', () => { it('should return an error if send invalid metadata directive header', async () => { try { - await s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: copySource, - MetadataDirective: 'copyHalf' })); + await s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: copySource, + MetadataDirective: 'copyHalf', + }), + ); assert.fail('Expected error'); } catch (err) { checkError(err, 'InvalidArgument'); @@ -586,63 +758,83 @@ describe('Object Version Copy', () => { await otherAccountBucketUtility.deleteOne(otherAccountBucket); }); - it('should not allow an account without read permission on the ' + - 'source object to copy the object', async () => { - try { - await otherAccountS3.send(new CopyObjectCommand({ Bucket: otherAccountBucket, - Key: otherAccountKey, - CopySource: copySource })); - assert.fail('Expected error'); - } catch (err) { - checkError(err, 'AccessDenied'); - } - }); - - it('should not allow an account without write permission on the ' + - 'destination bucket to copy the object', async () => { - await otherAccountS3.send(new PutObjectCommand({ Bucket: otherAccountBucket, - Key: otherAccountKey, - Body: '' })); - try { - await otherAccountS3.send(new CopyObjectCommand({ Bucket: destBucketName, - Key: destObjName, - CopySource: `${otherAccountBucket}/${otherAccountKey}` })); - assert.fail('Expected error'); - } catch (err) { - checkError(err, 'AccessDenied'); - } - }); - - it('should allow an account with read permission on the ' + - 'source object and write permission on the destination ' + - 'bucket to copy the object', async () => { - await s3.send(new PutObjectAclCommand({ Bucket: sourceBucketName, - Key: sourceObjName, - ACL: 'public-read', - VersionId: versionId })); - await otherAccountS3.send(new CopyObjectCommand({ Bucket: otherAccountBucket, - Key: otherAccountKey, - CopySource: copySource })); - }); - }); - - it('If-Match: returns no error when ETag match, with double quotes ' + - 'around ETag', async () => { + it( + 'should not allow an account without read permission on the ' + 'source object to copy the object', + async () => { + try { + await otherAccountS3.send( + new CopyObjectCommand({ + Bucket: otherAccountBucket, + Key: otherAccountKey, + CopySource: copySource, + }), + ); + assert.fail('Expected error'); + } catch (err) { + checkError(err, 'AccessDenied'); + } + }, + ); + + it( + 'should not allow an account without write permission on the ' + + 'destination bucket to copy the object', + async () => { + await otherAccountS3.send( + new PutObjectCommand({ Bucket: otherAccountBucket, Key: otherAccountKey, Body: '' }), + ); + try { + await otherAccountS3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${otherAccountBucket}/${otherAccountKey}`, + }), + ); + assert.fail('Expected error'); + } catch (err) { + checkError(err, 'AccessDenied'); + } + }, + ); + + it( + 'should allow an account with read permission on the ' + + 'source object and write permission on the destination ' + + 'bucket to copy the object', + async () => { + await s3.send( + new PutObjectAclCommand({ + Bucket: sourceBucketName, + Key: sourceObjName, + ACL: 'public-read', + VersionId: versionId, + }), + ); + await otherAccountS3.send( + new CopyObjectCommand({ + Bucket: otherAccountBucket, + Key: otherAccountKey, + CopySource: copySource, + }), + ); + }, + ); + }); + + it('If-Match: returns no error when ETag match, with double quotes ' + 'around ETag', async () => { await requestCopy({ CopySourceIfMatch: etag }); }); - it('If-Match: returns no error when one of ETags match, with double ' + - 'quotes around ETag', async () => { + it('If-Match: returns no error when one of ETags match, with double ' + 'quotes around ETag', async () => { await requestCopy({ CopySourceIfMatch: `non-matching,${etag}` }); }); - it('If-Match: returns no error when ETag match, without double ' + - 'quotes around ETag', async () => { + it('If-Match: returns no error when ETag match, without double ' + 'quotes around ETag', async () => { await requestCopy({ CopySourceIfMatch: etagTrim }); }); - it('If-Match: returns no error when one of ETags match, without ' + - 'double quotes around ETag', async () => { + it('If-Match: returns no error when one of ETags match, without ' + 'double quotes around ETag', async () => { await requestCopy({ CopySourceIfMatch: `non-matching,${etagTrim}` }); }); @@ -667,8 +859,7 @@ describe('Object Version Copy', () => { await requestCopy({ CopySourceIfNoneMatch: 'non-matching,non-matching-either' }); }); - it('If-None-Match: returns NotModified when ETag match, with double ' + - 'quotes around ETag', async () => { + it('If-None-Match: returns NotModified when ETag match, with double ' + 'quotes around ETag', async () => { try { await requestCopy({ CopySourceIfNoneMatch: etag }); assert.fail('Expected error'); @@ -677,18 +868,19 @@ describe('Object Version Copy', () => { } }); - it('If-None-Match: returns NotModified when one of ETags match, with ' + - 'double quotes around ETag', async () => { - try { - await requestCopy({ CopySourceIfNoneMatch: `non-matching,${etag}` }); - assert.fail('Expected error'); - } catch (err) { - checkError(err, 'PreconditionFailed'); - } - }); + it( + 'If-None-Match: returns NotModified when one of ETags match, with ' + 'double quotes around ETag', + async () => { + try { + await requestCopy({ CopySourceIfNoneMatch: `non-matching,${etag}` }); + assert.fail('Expected error'); + } catch (err) { + checkError(err, 'PreconditionFailed'); + } + }, + ); - it('If-None-Match: returns NotModified when ETag match, without ' + - 'double quotes around ETag', async () => { + it('If-None-Match: returns NotModified when ETag match, without ' + 'double quotes around ETag', async () => { try { await requestCopy({ CopySourceIfNoneMatch: etagTrim }); assert.fail('Expected error'); @@ -697,24 +889,24 @@ describe('Object Version Copy', () => { } }); - it('If-None-Match: returns NotModified when one of ETags match, ' + - 'without double quotes around ETag', async () => { - try { - await requestCopy({ CopySourceIfNoneMatch: `non-matching,${etagTrim}` }); - assert.fail('Expected error'); - } catch (err) { - checkError(err, 'PreconditionFailed'); - } - }); + it( + 'If-None-Match: returns NotModified when one of ETags match, ' + 'without double quotes around ETag', + async () => { + try { + await requestCopy({ CopySourceIfNoneMatch: `non-matching,${etagTrim}` }); + assert.fail('Expected error'); + } catch (err) { + checkError(err, 'PreconditionFailed'); + } + }, + ); - it('If-Modified-Since: returns no error if Last modified date is ' + - 'greater', async () => { + it('If-Modified-Since: returns no error if Last modified date is ' + 'greater', async () => { await requestCopy({ CopySourceIfModifiedSince: dateFromNow(-1) }); }); // Skipping this test, because real AWS does not provide error as // expected - it.skip('If-Modified-Since: returns NotModified if Last modified ' + - 'date is lesser', async () => { + it.skip('If-Modified-Since: returns NotModified if Last modified ' + 'date is lesser', async () => { try { await requestCopy({ CopySourceIfModifiedSince: dateFromNow(1) }); assert.fail('Expected error'); @@ -723,8 +915,7 @@ describe('Object Version Copy', () => { } }); - it('If-Modified-Since: returns NotModified if Last modified '+ - 'date is equal', async () => { + it('If-Modified-Since: returns NotModified if Last modified ' + 'date is equal', async () => { try { await requestCopy({ CopySourceIfModifiedSince: dateConvert(lastModified) }); assert.fail('Expected error'); @@ -733,18 +924,15 @@ describe('Object Version Copy', () => { } }); - it('If-Unmodified-Since: returns no error when lastModified date is ' + - 'greater', async () => { + it('If-Unmodified-Since: returns no error when lastModified date is ' + 'greater', async () => { await requestCopy({ CopySourceIfUnmodifiedSince: dateFromNow(1) }); }); - it('If-Unmodified-Since: returns no error when lastModified ' + - 'date is equal', async () => { + it('If-Unmodified-Since: returns no error when lastModified ' + 'date is equal', async () => { await requestCopy({ CopySourceIfUnmodifiedSince: dateConvert(lastModified) }); }); - it('If-Unmodified-Since: returns PreconditionFailed when ' + - 'lastModified date is lesser', async () => { + it('If-Unmodified-Since: returns PreconditionFailed when ' + 'lastModified date is lesser', async () => { try { await requestCopy({ CopySourceIfUnmodifiedSince: dateFromNow(-1) }); assert.fail('Expected error'); @@ -753,10 +941,12 @@ describe('Object Version Copy', () => { } }); - it('If-Match & If-Unmodified-Since: returns no error when match Etag ' + - 'and lastModified is greater', async () => { - await requestCopy({ CopySourceIfMatch: etagTrim, CopySourceIfUnmodifiedSince: dateFromNow(-1) }); - }); + it( + 'If-Match & If-Unmodified-Since: returns no error when match Etag ' + 'and lastModified is greater', + async () => { + await requestCopy({ CopySourceIfMatch: etagTrim, CopySourceIfUnmodifiedSince: dateFromNow(-1) }); + }, + ); it('If-Match match & If-Unmodified-Since match', async () => { await requestCopy({ CopySourceIfMatch: etagTrim, CopySourceIfUnmodifiedSince: dateFromNow(1) }); @@ -775,7 +965,8 @@ describe('Object Version Copy', () => { try { await requestCopy({ CopySourceIfMatch: 'non-matching', - CopySourceIfUnmodifiedSince: dateFromNow(1) }); + CopySourceIfUnmodifiedSince: dateFromNow(1), + }); assert.fail('Expected error'); } catch (err) { checkError(err, 'PreconditionFailed'); @@ -791,14 +982,16 @@ describe('Object Version Copy', () => { it('If-Match match & If-Modified-Since match', async () => { await requestCopy({ CopySourceIfMatch: etagTrim, - CopySourceIfModifiedSince: dateFromNow(-1) }); + CopySourceIfModifiedSince: dateFromNow(-1), + }); }); it('If-Match not match & If-Modified-Since not match', async () => { try { await requestCopy({ CopySourceIfMatch: 'non-matching', - CopySourceIfModifiedSince: dateFromNow(1) }); + CopySourceIfModifiedSince: dateFromNow(1), + }); assert.fail('Expected error'); } catch (err) { checkError(err, 'PreconditionFailed'); @@ -809,30 +1002,36 @@ describe('Object Version Copy', () => { try { await requestCopy({ CopySourceIfMatch: 'non-matching', - CopySourceIfModifiedSince: dateFromNow(-1) }); + CopySourceIfModifiedSince: dateFromNow(-1), + }); assert.fail('Expected error'); } catch (err) { checkError(err, 'PreconditionFailed'); } }); - it('If-None-Match & If-Modified-Since: returns NotModified when Etag ' + - 'does not match and lastModified is greater', async () => { - try { - await requestCopy({ - CopySourceIfNoneMatch: etagTrim, - CopySourceIfModifiedSince: dateFromNow(-1) }); - assert.fail('Expected error'); - } catch (err) { - checkError(err, 'PreconditionFailed'); - } - }); + it( + 'If-None-Match & If-Modified-Since: returns NotModified when Etag ' + + 'does not match and lastModified is greater', + async () => { + try { + await requestCopy({ + CopySourceIfNoneMatch: etagTrim, + CopySourceIfModifiedSince: dateFromNow(-1), + }); + assert.fail('Expected error'); + } catch (err) { + checkError(err, 'PreconditionFailed'); + } + }, + ); it('If-None-Match not match & If-Modified-Since not match', async () => { try { await requestCopy({ CopySourceIfNoneMatch: etagTrim, - CopySourceIfModifiedSince: dateFromNow(1) }); + CopySourceIfModifiedSince: dateFromNow(1), + }); assert.fail('Expected error'); } catch (err) { checkError(err, 'PreconditionFailed'); @@ -842,7 +1041,8 @@ describe('Object Version Copy', () => { it('If-None-Match match & If-Modified-Since match', async () => { await requestCopy({ CopySourceIfNoneMatch: 'non-matching', - CopySourceIfModifiedSince: dateFromNow(-1) }); + CopySourceIfModifiedSince: dateFromNow(-1), + }); }); // Skipping this test, because real AWS does not provide error as @@ -851,7 +1051,8 @@ describe('Object Version Copy', () => { try { await requestCopy({ CopySourceIfNoneMatch: 'non-matching', - CopySourceIfModifiedSince: dateFromNow(1) }); + CopySourceIfModifiedSince: dateFromNow(1), + }); assert.fail('Expected error'); } catch (err) { checkError(err, 'PreconditionFailed'); @@ -861,14 +1062,16 @@ describe('Object Version Copy', () => { it('If-None-Match match & If-Unmodified-Since match', async () => { await requestCopy({ CopySourceIfNoneMatch: 'non-matching', - CopySourceIfUnmodifiedSince: dateFromNow(1) }); + CopySourceIfUnmodifiedSince: dateFromNow(1), + }); }); it('If-None-Match match & If-Unmodified-Since not match', async () => { try { await requestCopy({ CopySourceIfNoneMatch: 'non-matching', - CopySourceIfUnmodifiedSince: dateFromNow(-1) }); + CopySourceIfUnmodifiedSince: dateFromNow(-1), + }); assert.fail('Expected error'); } catch (err) { checkError(err, 'PreconditionFailed'); @@ -879,7 +1082,8 @@ describe('Object Version Copy', () => { try { await requestCopy({ CopySourceIfNoneMatch: etagTrim, - CopySourceIfUnmodifiedSince: dateFromNow(1) }); + CopySourceIfUnmodifiedSince: dateFromNow(1), + }); assert.fail('Expected error'); } catch (err) { checkError(err, 'PreconditionFailed'); @@ -890,7 +1094,8 @@ describe('Object Version Copy', () => { try { await requestCopy({ CopySourceIfNoneMatch: etagTrim, - CopySourceIfUnmodifiedSince: dateFromNow(-1) }); + CopySourceIfUnmodifiedSince: dateFromNow(-1), + }); assert.fail('Expected error'); } catch (err) { checkError(err, 'PreconditionFailed'); diff --git a/tests/functional/aws-node-sdk/test/versioning/objectDelete.js b/tests/functional/aws-node-sdk/test/versioning/objectDelete.js index 34c00968bf..869c99b800 100644 --- a/tests/functional/aws-node-sdk/test/versioning/objectDelete.js +++ b/tests/functional/aws-node-sdk/test/versioning/objectDelete.js @@ -13,11 +13,7 @@ const { const withV4 = require('../support/withV4'); const BucketUtility = require('../../lib/utility/bucket-util'); -const { - versioningSuspended, - versioningEnabled, - removeAllVersions, -} = require('../../lib/utility/versioning-util.js'); +const { versioningSuspended, versioningEnabled, removeAllVersions } = require('../../lib/utility/versioning-util.js'); const { promisify } = require('util'); const removeAllVersionsPromise = promisify(removeAllVersions); @@ -26,9 +22,9 @@ const bucket = `versioning-bucket-${Date.now()}`; const key = 'anObject'; // formats differ for AWS and S3, use respective sample ids to obtain // correct error response in tests -const nonExistingId = process.env.AWS_ON_AIR ? - 'MhhyTHhmZ4cxSi4Y9SMe5P7UJAz7HLJ9' : - '3939393939393939393936493939393939393939756e6437'; +const nonExistingId = process.env.AWS_ON_AIR + ? 'MhhyTHhmZ4cxSi4Y9SMe5P7UJAz7HLJ9' + : '3939393939393939393936493939393939393939756e6437'; describe('delete marker creation in bucket with null version', () => { withV4(sigCfg => { @@ -38,11 +34,13 @@ describe('delete marker creation in bucket with null version', () => { beforeEach(async () => { await s3.send(new CreateBucketCommand({ Bucket: bucket })); - await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: nullVersionBody, - })); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: nullVersionBody, + }), + ); }); afterEach(async () => { @@ -58,10 +56,12 @@ describe('delete marker creation in bucket with null version', () => { }); it('should keep the null version if versioning enabled', async () => { - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningEnabled, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningEnabled, + }), + ); const listData = await s3.send(new ListObjectVersionsCommand({ Bucket: bucket })); assert.strictEqual(listData.Versions.length, 1); @@ -77,12 +77,13 @@ describe('delete marker creation in bucket with null version', () => { assert.strictEqual(listData2.DeleteMarkers[0].VersionId, deleteData.VersionId); }); - it('delete marker overwrites null version if versioning suspended', - async () => { - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningSuspended, - })); + it('delete marker overwrites null version if versioning suspended', async () => { + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningSuspended, + }), + ); const listData = await s3.send(new ListObjectVersionsCommand({ Bucket: bucket })); assert.strictEqual(listData.Versions.length, 1); @@ -124,36 +125,42 @@ describe('aws-node-sdk test delete object', () => { } }); - it('delete non existent object should not create a delete marker', - async () => { - const res = await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: `${key}000`, - })); + it('delete non existent object should not create a delete marker', async () => { + const res = await s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: `${key}000`, + }), + ); assert.strictEqual(res.DeleteMarker, undefined); assert.strictEqual(res.VersionId, undefined); }); it('creating non-versioned object', async () => { - const res = await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - })); + const res = await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + }), + ); assert.equal(res.VersionId, undefined); }); - it('delete in non-versioned bucket should not create delete marker', - async () => { - const putRes = await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - })); + it('delete in non-versioned bucket should not create delete marker', async () => { + const putRes = await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + }), + ); assert.equal(putRes.VersionId, undefined); - const deleteRes = await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: `${key}2`, - })); + const deleteRes = await s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: `${key}2`, + }), + ); assert.strictEqual(deleteRes.DeleteMarker, undefined); assert.strictEqual(deleteRes.VersionId, undefined); }); @@ -168,50 +175,60 @@ describe('aws-node-sdk test delete object', () => { await s3.send(new PutBucketVersioningCommand(params)); }); - it('should not send back error for non-existing key (specific version)', - async () => { - await s3.send(new DeleteObjectCommand({ + it('should not send back error for non-existing key (specific version)', async () => { + await s3.send( + new DeleteObjectCommand({ Bucket: bucket, Key: `${key}3`, VersionId: 'null', - })); - }); + }), + ); + }); it('delete non existent object should create a delete marker', async () => { - const res = await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: `${key}2`, - })); + const res = await s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: `${key}2`, + }), + ); assert.strictEqual(res.DeleteMarker, true); assert.notEqual(res.VersionId, undefined); - const res2 = await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: `${key}2`, - })); + const res2 = await s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: `${key}2`, + }), + ); assert.strictEqual(res2.DeleteMarker, true); assert.notEqual(res2.VersionId, res.VersionId); - await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: `${key}2`, - VersionId: res.VersionId, - })); + await s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: `${key}2`, + VersionId: res.VersionId, + }), + ); - await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: `${key}2`, - VersionId: res2.VersionId, - })); + await s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: `${key}2`, + VersionId: res2.VersionId, + }), + ); }); - it('delete non existent version should not create delete marker', - async () => { - const res = await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: nonExistingId, - })); + it('delete non existent version should not create delete marker', async () => { + const res = await s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: nonExistingId, + }), + ); assert.strictEqual(res.VersionId, nonExistingId); const listRes = await s3.send(new ListObjectVersionsCommand({ Bucket: bucket })); @@ -219,69 +236,84 @@ describe('aws-node-sdk test delete object', () => { }); it('put a version to the object', async () => { - const res = await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: 'test', - })); + const res = await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: 'test', + }), + ); versionIds.push('null'); versionIds.push(res.VersionId); assert.notEqual(res.VersionId, undefined); }); it('should create a delete marker', async () => { - const res = await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: key, - })); + const res = await s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: key, + }), + ); assert.strictEqual(res.DeleteMarker, true); assert.strictEqual( versionIds.find(item => item === res.VersionId), - undefined); + undefined, + ); versionIds.push(res.VersionId); }); it('should return 404 with a delete marker', done => { - s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - })).then(() => { - done(new Error('should return 404')); - }).catch(err => { - assert.strictEqual(err.Code, 'NoSuchKey'); - done(); - }); + s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + }), + ) + .then(() => { + done(new Error('should return 404')); + }) + .catch(err => { + assert.strictEqual(err.Code, 'NoSuchKey'); + done(); + }); }); it('should delete the null version', async () => { const version = versionIds.shift(); - const res = await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: version, - })); + const res = await s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: version, + }), + ); assert.strictEqual(res.VersionId, version); assert.equal(res.DeleteMarker, undefined); }); it('should delete the versioned object', async () => { const version = versionIds.shift(); - const res = await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: version, - })); + const res = await s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: version, + }), + ); assert.strictEqual(res.VersionId, version); assert.equal(res.DeleteMarker, undefined); }); it('should delete the delete-marker version', async () => { const version = versionIds.shift(); - const res = await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: version, - })); + const res = await s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: version, + }), + ); assert.strictEqual(res.VersionId, version); assert.equal(res.DeleteMarker, true); // In AWS SDK v3, the delete marker flag is sufficient for validation @@ -289,22 +321,26 @@ describe('aws-node-sdk test delete object', () => { }); it('put a new version', async () => { - const res = await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: 'test', - })); + const res = await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: 'test', + }), + ); versionIds.push(res.VersionId); assert.notEqual(res.VersionId, undefined); }); it('get the null version', async () => { try { - await s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: 'null', - })); + await s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: 'null', + }), + ); throw new Error('should send back an error'); } catch (err) { if (err.Code !== 'NoSuchVersion') { @@ -324,32 +360,40 @@ describe('aws-node-sdk test delete object', () => { }); it('delete non existent object should create a delete marker', async () => { - const res = await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: `${key}2`, - })); + const res = await s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: `${key}2`, + }), + ); assert.strictEqual(res.DeleteMarker, true); assert.notEqual(res.VersionId, undefined); - const res2 = await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: `${key}2`, - })); + const res2 = await s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: `${key}2`, + }), + ); assert.strictEqual(res2.DeleteMarker, true); assert.strictEqual(res2.VersionId, res.VersionId); - await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: `${key}2`, - VersionId: res.VersionId, - })); + await s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: `${key}2`, + VersionId: res.VersionId, + }), + ); }); it('should put a new delete marker', async () => { - const res = await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: key, - })); + const res = await s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: key, + }), + ); assert.strictEqual(res.DeleteMarker, true); assert.strictEqual(res.VersionId, 'null'); }); @@ -365,27 +409,33 @@ describe('aws-node-sdk test delete object', () => { }); it('should get the null version', done => { - s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: 'null', - })).then(() => { - done('should return an error'); - }).catch(err => { - if (err.Code !== 'MethodNotAllowed') { - return done(err); - } else { - return done(); - } - }); + s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: 'null', + }), + ) + .then(() => { + done('should return an error'); + }) + .catch(err => { + if (err.Code !== 'MethodNotAllowed') { + return done(err); + } else { + return done(); + } + }); }); it('put a new version to store the null version', async () => { - const res = await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: 'test', - })); + const res = await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: 'test', + }), + ); versionIds.push(res.VersionId); }); @@ -400,11 +450,13 @@ describe('aws-node-sdk test delete object', () => { }); it('put null version', async () => { - const res = await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: 'test-null-version', - })); + const res = await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: 'test-null-version', + }), + ); assert.strictEqual(res.VersionId, undefined); }); @@ -419,95 +471,115 @@ describe('aws-node-sdk test delete object', () => { }); it('should get the null version', async () => { - const res = await s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - })); + const res = await s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + }), + ); const body = await res.Body.transformToString(); assert.strictEqual(body, 'test-null-version'); }); it('should add a delete marker', async () => { - const res = await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: key, - })); + const res = await s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: key, + }), + ); assert.strictEqual(res.DeleteMarker, true); versionIds.push(res.VersionId); }); it('should get the null version', async () => { - const res = await s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: 'null', - })); + const res = await s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: 'null', + }), + ); const body = await res.Body.transformToString(); assert.strictEqual(body, 'test-null-version'); }); it('should add a delete marker', async () => { - const res = await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: key, - })); + const res = await s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: key, + }), + ); assert.strictEqual(res.DeleteMarker, true); assert.strictEqual( versionIds.find(item => item === res.VersionId), - undefined); + undefined, + ); versionIds.push(res.VersionId); }); it('should set the null version as master', async () => { let version = versionIds.pop(); - const res = await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: version, - })); + const res = await s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: version, + }), + ); assert.strictEqual(res.VersionId, version); assert.strictEqual(res.DeleteMarker, true); - + version = versionIds.pop(); - const res2 = await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: version, - })); + const res2 = await s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: version, + }), + ); assert.strictEqual(res2.VersionId, version); assert.strictEqual(res2.DeleteMarker, true); - - const getRes = await s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - })); + + const getRes = await s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + }), + ); const body = await getRes.Body.transformToString(); assert.strictEqual(body, 'test-null-version'); }); it('should delete null version', async () => { - const res = await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: 'null', - })); + const res = await s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: 'null', + }), + ); assert.strictEqual(res.VersionId, 'null'); - - const getRes = await s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - })); - assert.strictEqual(getRes.VersionId, - versionIds[versionIds.length - 1]); + + const getRes = await s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + }), + ); + assert.strictEqual(getRes.VersionId, versionIds[versionIds.length - 1]); }); it('should be able to delete the bucket', async () => { for (const id of versionIds) { - const res = await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: id, - })); + const res = await s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: id, + }), + ); assert.strictEqual(res.VersionId, id); } await s3.send(new DeleteBucketCommand({ Bucket: bucket })); @@ -535,11 +607,13 @@ describe('aws-node-sdk test concurrent version-specific deletes with null', () = }); it('creating non-versioned object', async () => { - const res = await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: 'null-body', - })); + const res = await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: 'null-body', + }), + ); assert.equal(res.VersionId, undefined); }); @@ -556,11 +630,15 @@ describe('aws-node-sdk test concurrent version-specific deletes with null', () = it('put 5 new versions to the object', async () => { const promises = []; for (let i = 0; i < 5; i++) { - promises.push(s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: `test-body-${i}`, - }))); + promises.push( + s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: `test-body-${i}`, + }), + ), + ); } await Promise.all(promises); }); @@ -570,16 +648,18 @@ describe('aws-node-sdk test concurrent version-specific deletes with null', () = assert.strictEqual(res.DeleteMarkers, undefined); assert.strictEqual(res.Versions.length, 6); assert.strictEqual(res.Versions[5].VersionId, 'null'); - - await s3.send(new DeleteObjectsCommand({ - Bucket: bucket, - Delete: { - Objects: res.Versions.slice(0, 5).map(item => ({ - Key: item.Key, - VersionId: item.VersionId, - })), - }, - })); + + await s3.send( + new DeleteObjectsCommand({ + Bucket: bucket, + Delete: { + Objects: res.Versions.slice(0, 5).map(item => ({ + Key: item.Key, + VersionId: item.VersionId, + })), + }, + }), + ); }); it('list versions should return a list with just the null version', async () => { diff --git a/tests/functional/aws-node-sdk/test/versioning/objectDeleteTagging.js b/tests/functional/aws-node-sdk/test/versioning/objectDeleteTagging.js index 70d039dc55..07a8b3d4bd 100644 --- a/tests/functional/aws-node-sdk/test/versioning/objectDeleteTagging.js +++ b/tests/functional/aws-node-sdk/test/versioning/objectDeleteTagging.js @@ -18,10 +18,7 @@ const objectName = 'testtaggingobject'; const invalidId = 'invalidIdWithMoreThan40BytesAndThatIsNotLongEnoughYet'; -const { - removeAllVersions, - versioningEnabled, -} = require('../../lib/utility/versioning-util'); +const { removeAllVersions, versioningEnabled } = require('../../lib/utility/versioning-util'); function _checkError(err, code, statusCode) { assert(err, 'Expected error but found none'); @@ -29,16 +26,15 @@ function _checkError(err, code, statusCode) { assert.strictEqual(err.$metadata?.httpStatusCode, statusCode); } - describe('Delete object tagging with versioning', () => { withV4(sigCfg => { const bucketUtil = new BucketUtility('default', sigCfg); const s3 = bucketUtil.s3; - + beforeEach(async () => { await s3.send(new CreateBucketCommand({ Bucket: bucketName })); }); - + afterEach(async () => { await removeAllVersions({ Bucket: bucketName }); await bucketUtil.empty(bucketName); @@ -46,172 +42,227 @@ describe('Delete object tagging with versioning', () => { }); it('should be able to delete tag set with versioning', async () => { - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: versioningEnabled - })); - - const putObjectResult = await s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: versioningEnabled, + }), + ); + + const putObjectResult = await s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); const versionId = putObjectResult.VersionId; - await s3.send(new PutObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - VersionId: versionId, - Tagging: { - TagSet: [{ - Key: 'key1', - Value: 'value1', - }] - }, - })); - - const deleteResult = await s3.send(new DeleteObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - VersionId: versionId, - })); + await s3.send( + new PutObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + VersionId: versionId, + Tagging: { + TagSet: [ + { + Key: 'key1', + Value: 'value1', + }, + ], + }, + }), + ); + + const deleteResult = await s3.send( + new DeleteObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + VersionId: versionId, + }), + ); assert.strictEqual(deleteResult.VersionId, versionId); }); - it('should not create version deleting object tags on a ' + - ' version-enabled bucket where no version id is specified ', async () => { - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: versioningEnabled - })); - - const putObjectResult = await s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName - })); - const versionId = putObjectResult.VersionId; - - await s3.send(new PutObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - VersionId: versionId, - Tagging: { - TagSet: [{ - Key: 'key1', - Value: 'value1', - }] - }, - })); - - await s3.send(new DeleteObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - })); - - await checkOneVersion(s3, bucketName, versionId); - }); - - it('should be able to delete tag set with a version of id "null"', - async () => { - await s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName - })); + it( + 'should not create version deleting object tags on a ' + + ' version-enabled bucket where no version id is specified ', + async () => { + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: versioningEnabled, + }), + ); + + const putObjectResult = await s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); + const versionId = putObjectResult.VersionId; + + await s3.send( + new PutObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + VersionId: versionId, + Tagging: { + TagSet: [ + { + Key: 'key1', + Value: 'value1', + }, + ], + }, + }), + ); + + await s3.send( + new DeleteObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); + + await checkOneVersion(s3, bucketName, versionId); + }, + ); + + it('should be able to delete tag set with a version of id "null"', async () => { + await s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: versioningEnabled - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: versioningEnabled, + }), + ); - const deleteResult = await s3.send(new DeleteObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - VersionId: 'null', - })); + const deleteResult = await s3.send( + new DeleteObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + VersionId: 'null', + }), + ); assert.strictEqual(deleteResult.VersionId, 'null'); }); - it('should return InvalidArgument deleting tag set with a non ' + - 'existing version id', async () => { - await s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName - })); - - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: versioningEnabled - })); - - try { - await s3.send(new DeleteObjectTaggingCommand({ + it('should return InvalidArgument deleting tag set with a non ' + 'existing version id', async () => { + await s3.send( + new PutObjectCommand({ Bucket: bucketName, Key: objectName, - VersionId: invalidId, - })); - assert.fail('Expected InvalidArgument error'); - } catch (err) { - _checkError(err, 'InvalidArgument', 400); - } - }); - - it('should return 405 MethodNotAllowed deleting tag set without ' + - 'version id if version specified is a delete marker', async () => { - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: versioningEnabled - })); - - await s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName - })); + }), + ); - await s3.send(new DeleteObjectCommand({ - Bucket: bucketName, - Key: objectName - })); - - try { - await s3.send(new DeleteObjectTaggingCommand({ + await s3.send( + new PutBucketVersioningCommand({ Bucket: bucketName, - Key: objectName, - })); - assert.fail('Expected MethodNotAllowed error'); - } catch (err) { - _checkError(err, 'MethodNotAllowed', 405); - } - }); - - it('should return 405 MethodNotAllowed deleting tag set with ' + - 'version id if version specified is a delete marker', async () => { - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: versioningEnabled - })); - - await s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName - })); - - const deleteResult = await s3.send(new DeleteObjectCommand({ - Bucket: bucketName, - Key: objectName - })); - const versionId = deleteResult.VersionId; + VersioningConfiguration: versioningEnabled, + }), + ); try { - await s3.send(new DeleteObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - VersionId: versionId, - })); - assert.fail('Expected MethodNotAllowed error'); + await s3.send( + new DeleteObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + VersionId: invalidId, + }), + ); + assert.fail('Expected InvalidArgument error'); } catch (err) { - _checkError(err, 'MethodNotAllowed', 405); + _checkError(err, 'InvalidArgument', 400); } }); + + it( + 'should return 405 MethodNotAllowed deleting tag set without ' + + 'version id if version specified is a delete marker', + async () => { + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: versioningEnabled, + }), + ); + + await s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); + + await s3.send( + new DeleteObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); + + try { + await s3.send( + new DeleteObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); + assert.fail('Expected MethodNotAllowed error'); + } catch (err) { + _checkError(err, 'MethodNotAllowed', 405); + } + }, + ); + + it( + 'should return 405 MethodNotAllowed deleting tag set with ' + + 'version id if version specified is a delete marker', + async () => { + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: versioningEnabled, + }), + ); + + await s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); + + const deleteResult = await s3.send( + new DeleteObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); + const versionId = deleteResult.VersionId; + + try { + await s3.send( + new DeleteObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + VersionId: versionId, + }), + ); + assert.fail('Expected MethodNotAllowed error'); + } catch (err) { + _checkError(err, 'MethodNotAllowed', 405); + } + }, + ); }); }); diff --git a/tests/functional/aws-node-sdk/test/versioning/objectGet.js b/tests/functional/aws-node-sdk/test/versioning/objectGet.js index 32eb7f4805..4b702e3838 100644 --- a/tests/functional/aws-node-sdk/test/versioning/objectGet.js +++ b/tests/functional/aws-node-sdk/test/versioning/objectGet.js @@ -3,35 +3,30 @@ const assert = require('assert'); const withV4 = require('../support/withV4'); const BucketUtility = require('../../lib/utility/bucket-util'); +const { removeAllVersions, versioningEnabled, versioningSuspended } = require('../../lib/utility/versioning-util.js'); const { - removeAllVersions, - versioningEnabled, - versioningSuspended, -} = require('../../lib/utility/versioning-util.js'); -const { CreateBucketCommand, + CreateBucketCommand, DeleteBucketCommand, PutBucketVersioningCommand, PutObjectCommand, GetObjectCommand, DeleteObjectCommand, - PutObjectTaggingCommand - } = require('@aws-sdk/client-s3'); + PutObjectTaggingCommand, +} = require('@aws-sdk/client-s3'); const key = 'objectKey'; // formats differ for AWS and S3, use respective sample ids to obtain // correct error response in tests -const nonExistingId = process.env.AWS_ON_AIR ? - 'MhhyTHhmZ4cxSi4Y9SMe5P7UJAz7HLJ9' : - '3939393939393939393936493939393939393939756e6437'; +const nonExistingId = process.env.AWS_ON_AIR + ? 'MhhyTHhmZ4cxSi4Y9SMe5P7UJAz7HLJ9' + : '3939393939393939393936493939393939393939756e6437'; function _assertError(err, statusCode, code) { - assert.notEqual(err, null, - 'Expected failure but got success'); + assert.notEqual(err, null, 'Expected failure but got success'); assert.strictEqual(err.name, code); assert.strictEqual(err.$metadata.httpStatusCode, statusCode); } - describe('get behavior on versioning-enabled bucket', () => { withV4(sigCfg => { const bucketUtil = new BucketUtility('default', sigCfg); @@ -42,10 +37,12 @@ describe('get behavior on versioning-enabled bucket', () => { beforeEach(async () => { bucket = `versioning-bucket-${Date.now()}`; await s3.send(new CreateBucketCommand({ Bucket: bucket })); - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningEnabled, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningEnabled, + }), + ); }); afterEach(async () => { @@ -61,21 +58,25 @@ describe('get behavior on versioning-enabled bucket', () => { }); it('should be able to get the object version', async () => { - const data = await s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: versionId, - })); + const data = await s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: versionId, + }), + ); assert.strictEqual(data.ContentLength, 0); }); it('it should return NoSuchVersion if try to get a non-existing object version', async () => { try { - await s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: nonExistingId, - })); + await s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: nonExistingId, + }), + ); assert.fail('Expected NoSuchVersion error but got success'); } catch (err) { _assertError(err, 404, 'NoSuchVersion'); @@ -84,11 +85,13 @@ describe('get behavior on versioning-enabled bucket', () => { it('it should return NoSuchVersion if try to get a non-existing null version', async () => { try { - await s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: 'null', - })); + await s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: 'null', + }), + ); assert.fail('Expected NoSuchVersion error but got success'); } catch (err) { _assertError(err, 404, 'NoSuchVersion'); @@ -96,24 +99,30 @@ describe('get behavior on versioning-enabled bucket', () => { }); it('it should return NoSuchVersion if try to get a deleted noncurrent null version', async () => { - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningSuspended, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningSuspended, + }), + ); await s3.send(new PutObjectCommand({ Bucket: bucket, Key: key })); - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningEnabled, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningEnabled, + }), + ); await s3.send(new PutObjectCommand({ Bucket: bucket, Key: key })); await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: key, VersionId: 'null' })); - + try { - await s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: 'null', - })); + await s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: 'null', + }), + ); assert.fail('Expected NoSuchVersion error but got success'); } catch (err) { _assertError(err, 404, 'NoSuchVersion'); @@ -123,7 +132,7 @@ describe('get behavior on versioning-enabled bucket', () => { describe('behavior when only version put is a delete marker', () => { let deleteVersionId; - + beforeEach(async () => { const deleteResult = await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: key })); deleteVersionId = deleteResult.VersionId; @@ -131,11 +140,13 @@ describe('get behavior on versioning-enabled bucket', () => { it('should not be able to get a delete marker', async () => { try { - await s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: deleteVersionId, - })); + await s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: deleteVersionId, + }), + ); assert.fail('Expected MethodNotAllowed error but got success'); } catch (err) { _assertError(err, 405, 'MethodNotAllowed'); @@ -145,25 +156,28 @@ describe('get behavior on versioning-enabled bucket', () => { } }); - it('it should return NoSuchKey if try to get object whose ' + - 'latest version is a delete marker', async () => { - try { - await s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - })); - assert.fail('Expected NoSuchKey error but got success'); - } catch (err) { - _assertError(err, 404, 'NoSuchKey'); - } - }); + it( + 'it should return NoSuchKey if try to get object whose ' + 'latest version is a delete marker', + async () => { + try { + await s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + }), + ); + assert.fail('Expected NoSuchKey error but got success'); + } catch (err) { + _assertError(err, 404, 'NoSuchKey'); + } + }, + ); }); - describe('behavior when put version with content then put delete ' + - 'marker', () => { + describe('behavior when put version with content then put delete ' + 'marker', () => { let putVersionId; let deleteVersionId; - + beforeEach(async () => { const putResult = await s3.send(new PutObjectCommand({ Bucket: bucket, Key: key })); putVersionId = putResult.VersionId; @@ -173,47 +187,53 @@ describe('get behavior on versioning-enabled bucket', () => { it('should not be able to get a delete marker', async () => { try { - await s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: deleteVersionId, - })); + await s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: deleteVersionId, + }), + ); assert.fail('Expected MethodNotAllowed error but got success'); } catch (err) { _assertError(err, 405, 'MethodNotAllowed'); } }); - it('should be able to get a version that was put prior to the ' + - 'delete marker', async () => { - const data = await s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: putVersionId - })); - assert.strictEqual(data.VersionId, putVersionId); - }); - - it('should return NoSuchKey if get object without version and ' + - 'latest version is a delete marker', - async () => { - try { - await s3.send(new GetObjectCommand({ + it('should be able to get a version that was put prior to the ' + 'delete marker', async () => { + const data = await s3.send( + new GetObjectCommand({ Bucket: bucket, Key: key, - })); - assert.fail('Expected NoSuchKey error but got success'); - } catch (err) { - _assertError(err, 404, 'NoSuchKey'); - } + VersionId: putVersionId, + }), + ); + assert.strictEqual(data.VersionId, putVersionId); }); + + it( + 'should return NoSuchKey if get object without version and ' + 'latest version is a delete marker', + async () => { + try { + await s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + }), + ); + assert.fail('Expected NoSuchKey error but got success'); + } catch (err) { + _assertError(err, 404, 'NoSuchKey'); + } + }, + ); }); describe('x-amz-tagging-count with versioning', () => { let params; let paramsTagging; let objectVersionId; - + beforeEach(async () => { params = { Bucket: bucket, @@ -235,9 +255,7 @@ describe('get behavior on versioning-enabled bucket', () => { objectVersionId = data.VersionId; }); - it('should not return "x-amz-tagging-count" if no tag ' + - 'associated with the object', - async () => { + it('should not return "x-amz-tagging-count" if no tag ' + 'associated with the object', async () => { params.VersionId = objectVersionId; const data = await s3.send(new GetObjectCommand(params)); assert.strictEqual(data.TagCount, undefined); @@ -249,15 +267,16 @@ describe('get behavior on versioning-enabled bucket', () => { await s3.send(new PutObjectTaggingCommand(paramsTagging)); }); - it('should return "x-amz-tagging-count" header that provides ' + - 'the count of number of tags associated with the object', - async () => { - params.VersionId = objectVersionId; - const data = await s3.send(new GetObjectCommand(params)); - assert.equal(data.TagCount, 1); - }); + it( + 'should return "x-amz-tagging-count" header that provides ' + + 'the count of number of tags associated with the object', + async () => { + params.VersionId = objectVersionId; + const data = await s3.send(new GetObjectCommand(params)); + assert.equal(data.TagCount, 1); + }, + ); }); }); }); }); - diff --git a/tests/functional/aws-node-sdk/test/versioning/objectGetAttributes.js b/tests/functional/aws-node-sdk/test/versioning/objectGetAttributes.js index bae4e16a1c..1943a47162 100644 --- a/tests/functional/aws-node-sdk/test/versioning/objectGetAttributes.js +++ b/tests/functional/aws-node-sdk/test/versioning/objectGetAttributes.js @@ -29,10 +29,12 @@ describe('Test get object attributes with versioning', () => { beforeEach(async () => { await s3.send(new CreateBucketCommand({ Bucket: bucket })); - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningEnabled, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningEnabled, + }), + ); }); afterEach(done => { @@ -40,28 +42,33 @@ describe('Test get object attributes with versioning', () => { if (err) { return done(err); } - return s3.send(new DeleteBucketCommand({ Bucket: bucket })) + return s3 + .send(new DeleteBucketCommand({ Bucket: bucket })) .then(() => done()) .catch(done); }); }); it('should return NoSuchVersion for non-existent versionId', async () => { - await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: body, - })); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: body, + }), + ); const fakeVersionId = '111111111111111111111111111111111111111175636f7270'; try { - await s3.send(new GetObjectAttributesCommand({ - Bucket: bucket, - Key: key, - VersionId: fakeVersionId, - ObjectAttributes: ['ETag'], - })); + await s3.send( + new GetObjectAttributesCommand({ + Bucket: bucket, + Key: key, + VersionId: fakeVersionId, + ObjectAttributes: ['ETag'], + }), + ); assert.fail('Expected NoSuchVersion error'); } catch (err) { assert.strictEqual(err.name, 'NoSuchVersion'); @@ -73,23 +80,29 @@ describe('Test get object attributes with versioning', () => { }); it('should return MethodNotAllowed for delete marker', async () => { - await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: body, - })); - - await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: key, - })); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: body, + }), + ); - try { - await s3.send(new GetObjectAttributesCommand({ + await s3.send( + new DeleteObjectCommand({ Bucket: bucket, Key: key, - ObjectAttributes: ['ETag'], - })); + }), + ); + + try { + await s3.send( + new GetObjectAttributesCommand({ + Bucket: bucket, + Key: key, + ObjectAttributes: ['ETag'], + }), + ); assert.fail('Expected MethodNotAllowed error'); } catch (err) { assert.strictEqual(err.name, 'MethodNotAllowed'); @@ -98,19 +111,23 @@ describe('Test get object attributes with versioning', () => { }); it('should return attributes for specific version', async () => { - const putResult = await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: body, - })); + const putResult = await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: body, + }), + ); const versionId = putResult.VersionId; - const data = await s3.send(new GetObjectAttributesCommand({ - Bucket: bucket, - Key: key, - VersionId: versionId, - ObjectAttributes: ['ETag', 'ObjectSize'], - })); + const data = await s3.send( + new GetObjectAttributesCommand({ + Bucket: bucket, + Key: key, + VersionId: versionId, + ObjectAttributes: ['ETag', 'ObjectSize'], + }), + ); assert.strictEqual(data.ETag, expectedMD5); assert.strictEqual(data.ObjectSize, body.length); @@ -118,18 +135,22 @@ describe('Test get object attributes with versioning', () => { }); it('should return VersionId for versioned object', async () => { - const putResult = await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: body, - })); + const putResult = await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: body, + }), + ); const versionId = putResult.VersionId; - const data = await s3.send(new GetObjectAttributesCommand({ - Bucket: bucket, - Key: key, - ObjectAttributes: ['ETag'], - })); + const data = await s3.send( + new GetObjectAttributesCommand({ + Bucket: bucket, + Key: key, + ObjectAttributes: ['ETag'], + }), + ); assert.strictEqual(data.VersionId, versionId); }); diff --git a/tests/functional/aws-node-sdk/test/versioning/objectGetTagging.js b/tests/functional/aws-node-sdk/test/versioning/objectGetTagging.js index 28c8559ca8..76c8b366f9 100644 --- a/tests/functional/aws-node-sdk/test/versioning/objectGetTagging.js +++ b/tests/functional/aws-node-sdk/test/versioning/objectGetTagging.js @@ -14,10 +14,7 @@ const { const withV4 = require('../support/withV4'); const BucketUtility = require('../../lib/utility/bucket-util'); -const { - removeAllVersions, - versioningEnabled, -} = require('../../lib/utility/versioning-util'); +const { removeAllVersions, versioningEnabled } = require('../../lib/utility/versioning-util'); const removeAllVersionsPromise = promisify(removeAllVersions); const bucketName = 'testtaggingbucket'; @@ -45,145 +42,275 @@ describe('Get object tagging with versioning', () => { }); it('should be able to get tag with versioning', done => { - const taggingConfig = { TagSet: [ - { - Key: 'key1', - Value: 'value1', - }] }; - async.waterfall([ - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: versioningEnabled, - })).then(() => next()).catch(next), - - next => s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName, - })).then(data => next(null, data.VersionId)).catch(next), - - (versionId, next) => s3.send(new PutObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - VersionId: versionId, - Tagging: taggingConfig, - })).then(() => next(null, versionId)).catch(next), - - (versionId, next) => s3.send(new GetObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - VersionId: versionId, - })).then(data => next(null, data, versionId)).catch(next), - ], (err, data, versionId) => { - assert.ifError(err, `Found unexpected err ${err}`); - assert.strictEqual(data.VersionId, versionId); - assert.deepStrictEqual(data.TagSet, taggingConfig.TagSet); - done(); - }); + const taggingConfig = { + TagSet: [ + { + Key: 'key1', + Value: 'value1', + }, + ], + }; + async.waterfall( + [ + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: versioningEnabled, + }), + ) + .then(() => next()) + .catch(next), + + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ) + .then(data => next(null, data.VersionId)) + .catch(next), + + (versionId, next) => + s3 + .send( + new PutObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + VersionId: versionId, + Tagging: taggingConfig, + }), + ) + .then(() => next(null, versionId)) + .catch(next), + + (versionId, next) => + s3 + .send( + new GetObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + VersionId: versionId, + }), + ) + .then(data => next(null, data, versionId)) + .catch(next), + ], + (err, data, versionId) => { + assert.ifError(err, `Found unexpected err ${err}`); + assert.strictEqual(data.VersionId, versionId); + assert.deepStrictEqual(data.TagSet, taggingConfig.TagSet); + done(); + }, + ); }); it('should be able to get tag with a version of id "null"', done => { - async.waterfall([ - next => s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName, - })).then(() => next()).catch(next), - - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: versioningEnabled, - })).then(() => next()).catch(next), - - next => s3.send(new GetObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - VersionId: 'null', - })).then(data => next(null, data)).catch(next), - ], (err, data) => { - assert.ifError(err, `Found unexpected err ${err}`); - assert.strictEqual(data.VersionId, 'null'); - done(); - }); - }); + async.waterfall( + [ + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ) + .then(() => next()) + .catch(next), - it('should return InvalidArgument getting tag with a non existing ' + - 'version id', done => { - async.waterfall([ - next => s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName, - })).then(() => next()).catch(next), - - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: versioningEnabled, - })).then(() => next()).catch(next), - - next => s3.send(new GetObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - VersionId: invalidId, - })).then(data => next(null, data)).catch(next), - ], err => { - _checkError(err, 'InvalidArgument', 400); - done(); - }); - }); + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: versioningEnabled, + }), + ) + .then(() => next()) + .catch(next), - it('should return 404 NoSuchKey getting tag without ' + - 'version id if version specified is a delete marker', done => { - async.waterfall([ - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: versioningEnabled, - })).then(() => next()).catch(next), - - next => s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName, - })).then(() => next()).catch(next), - - next => s3.send(new DeleteObjectCommand({ - Bucket: bucketName, - Key: objectName, - })).then(() => next()).catch(next), - - next => s3.send(new GetObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - })).then(data => next(null, data)).catch(next), - ], err => { - _checkError(err, 'NoSuchKey', 404); - done(); - }); + next => + s3 + .send( + new GetObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + VersionId: 'null', + }), + ) + .then(data => next(null, data)) + .catch(next), + ], + (err, data) => { + assert.ifError(err, `Found unexpected err ${err}`); + assert.strictEqual(data.VersionId, 'null'); + done(); + }, + ); }); - it('should return 405 MethodNotAllowed getting tag with ' + - 'version id if version specified is a delete marker', done => { - async.waterfall([ - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: versioningEnabled, - })).then(() => next()).catch(next), - - next => s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName, - })).then(() => next()).catch(next), - - next => s3.send(new DeleteObjectCommand({ - Bucket: bucketName, - Key: objectName, - })).then(data => next(null, data.VersionId)).catch(next), - - (versionId, next) => s3.send(new GetObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - VersionId: versionId, - })).then(data => next(null, data)).catch(next), - ], err => { - _checkError(err, 'MethodNotAllowed', 405); - done(); - }); + it('should return InvalidArgument getting tag with a non existing ' + 'version id', done => { + async.waterfall( + [ + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ) + .then(() => next()) + .catch(next), + + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: versioningEnabled, + }), + ) + .then(() => next()) + .catch(next), + + next => + s3 + .send( + new GetObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + VersionId: invalidId, + }), + ) + .then(data => next(null, data)) + .catch(next), + ], + err => { + _checkError(err, 'InvalidArgument', 400); + done(); + }, + ); }); + + it( + 'should return 404 NoSuchKey getting tag without ' + 'version id if version specified is a delete marker', + done => { + async.waterfall( + [ + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: versioningEnabled, + }), + ) + .then(() => next()) + .catch(next), + + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ) + .then(() => next()) + .catch(next), + + next => + s3 + .send( + new DeleteObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ) + .then(() => next()) + .catch(next), + + next => + s3 + .send( + new GetObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + }), + ) + .then(data => next(null, data)) + .catch(next), + ], + err => { + _checkError(err, 'NoSuchKey', 404); + done(); + }, + ); + }, + ); + + it( + 'should return 405 MethodNotAllowed getting tag with ' + + 'version id if version specified is a delete marker', + done => { + async.waterfall( + [ + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: versioningEnabled, + }), + ) + .then(() => next()) + .catch(next), + + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ) + .then(() => next()) + .catch(next), + + next => + s3 + .send( + new DeleteObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ) + .then(data => next(null, data.VersionId)) + .catch(next), + + (versionId, next) => + s3 + .send( + new GetObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + VersionId: versionId, + }), + ) + .then(data => next(null, data)) + .catch(next), + ], + err => { + _checkError(err, 'MethodNotAllowed', 405); + done(); + }, + ); + }, + ); }); }); diff --git a/tests/functional/aws-node-sdk/test/versioning/objectHead.js b/tests/functional/aws-node-sdk/test/versioning/objectHead.js index df8e448533..ad99ed6c45 100644 --- a/tests/functional/aws-node-sdk/test/versioning/objectHead.js +++ b/tests/functional/aws-node-sdk/test/versioning/objectHead.js @@ -13,11 +13,7 @@ const { const withV4 = require('../support/withV4'); const BucketUtility = require('../../lib/utility/bucket-util'); -const { - removeAllVersions, - versioningEnabled, - versioningSuspended, -} = require('../../lib/utility/versioning-util.js'); +const { removeAllVersions, versioningEnabled, versioningSuspended } = require('../../lib/utility/versioning-util.js'); const removeAllVersionsPromise = promisify(removeAllVersions); const data = ['foo1', 'foo2']; @@ -48,25 +44,26 @@ describe('put and head object with versioning', function testSuite() { await s3.send(new DeleteBucketCommand({ Bucket: bucket })); }); - it('should put and head a non-versioned object without including ' + - 'version ids in response headers', done => { - const params = { Bucket: bucket, Key: key }; - s3.send(new PutObjectCommand(params)) - .then(data => { - _assertNoError(null, 'putting object'); - assert.strictEqual(data.VersionId, undefined); - return s3.send(new HeadObjectCommand(params)); - }) - .then(data => { - _assertNoError(null, 'heading object'); - assert.strictEqual(data.VersionId, undefined); - done(); - }) - .catch(done); - }); + it( + 'should put and head a non-versioned object without including ' + 'version ids in response headers', + done => { + const params = { Bucket: bucket, Key: key }; + s3.send(new PutObjectCommand(params)) + .then(data => { + _assertNoError(null, 'putting object'); + assert.strictEqual(data.VersionId, undefined); + return s3.send(new HeadObjectCommand(params)); + }) + .then(data => { + _assertNoError(null, 'heading object'); + assert.strictEqual(data.VersionId, undefined); + done(); + }) + .catch(done); + }, + ); - it('version-specific head should still not return version id in ' + - 'response header', done => { + it('version-specific head should still not return version id in ' + 'response header', done => { const params = { Bucket: bucket, Key: key }; s3.send(new PutObjectCommand(params)) .then(data => { @@ -85,10 +82,12 @@ describe('put and head object with versioning', function testSuite() { describe('on a version-enabled bucket', () => { beforeEach(async () => { - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningEnabled, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningEnabled, + }), + ); }); it('should create a new version for an object', done => { @@ -101,8 +100,7 @@ describe('put and head object with versioning', function testSuite() { }) .then(data => { _assertNoError(null, 'heading object'); - assert.strictEqual(params.VersionId, data.VersionId, - 'version ids are not equal'); + assert.strictEqual(params.VersionId, data.VersionId, 'version ids are not equal'); done(); }) .catch(done); @@ -113,17 +111,21 @@ describe('put and head object with versioning', function testSuite() { const eTags = []; beforeEach(done => { - s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: data[0] - })) + s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: data[0], + }), + ) .then(data => { eTags.push(data.ETag); - return s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningEnabled, - })); + return s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningEnabled, + }), + ); }) .then(() => done()) .catch(done); @@ -135,8 +137,7 @@ describe('put and head object with versioning', function testSuite() { done(); }); - it('should head null version in versioning enabled bucket', - done => { + it('should head null version in versioning enabled bucket', done => { const paramsNull = { Bucket: bucket, Key: '/', @@ -157,21 +158,24 @@ describe('put and head object with versioning', function testSuite() { .then(data => { newVersion = data.VersionId; eTags.push(data.ETag); - return s3.send(new HeadObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: newVersion - })); + return s3.send( + new HeadObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: newVersion, + }), + ); }) .then(data => { - assert.strictEqual(data.VersionId, newVersion, - 'version ids are not equal'); + assert.strictEqual(data.VersionId, newVersion, 'version ids are not equal'); assert.strictEqual(data.ETag, eTags[1]); - return s3.send(new HeadObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: 'null' - })); + return s3.send( + new HeadObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: 'null', + }), + ); }) .then(data => { _assertNoError(null, 'heading null version'); @@ -182,8 +186,7 @@ describe('put and head object with versioning', function testSuite() { .catch(done); }); - it('should create new versions but still keep nullVersionId', - done => { + it('should create new versions but still keep nullVersionId', done => { const versionIds = []; const params = { Bucket: bucket, Key: key }; const paramsNull = { @@ -192,29 +195,35 @@ describe('put and head object with versioning', function testSuite() { VersionId: 'null', }; // create new versions - async.timesSeries(counter, (i, next) => { - s3.send(new PutObjectCommand(params)) - .then(data => { - versionIds.push(data.VersionId); - // head the 'null' version - return s3.send(new HeadObjectCommand(paramsNull)); - }) - .then(nullVerData => { - assert.strictEqual(nullVerData.ETag, eTags[0]); - assert.strictEqual(nullVerData.VersionId, 'null'); - next(); - }) - .catch(next); - }, done); + async.timesSeries( + counter, + (i, next) => { + s3.send(new PutObjectCommand(params)) + .then(data => { + versionIds.push(data.VersionId); + // head the 'null' version + return s3.send(new HeadObjectCommand(paramsNull)); + }) + .then(nullVerData => { + assert.strictEqual(nullVerData.ETag, eTags[0]); + assert.strictEqual(nullVerData.VersionId, 'null'); + next(); + }) + .catch(next); + }, + done, + ); }); }); describe('on version-suspended bucket', () => { beforeEach(async () => { - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningSuspended, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningSuspended, + }), + ); }); it('should not return version id for new object', done => { @@ -252,61 +261,73 @@ describe('put and head object with versioning', function testSuite() { VersionId: 'null', }; const eTags = []; - async.waterfall([ - callback => s3.send(new PutObjectCommand(params1)) - .then(data => { - _assertNoError(null, 'putting first object'); - assert.strictEqual(data.VersionId, undefined); - eTags.push(data.ETag); - callback(); - }) - .catch(callback), - callback => s3.send(new HeadObjectCommand(params)) - .then(data => { - _assertNoError(null, 'heading master version'); - assert.strictEqual(data.VersionId, 'null'); - assert.strictEqual(data.ETag, eTags[0], - 'wrong object data'); - callback(); - }) - .catch(callback), - callback => s3.send(new PutObjectCommand(params2)) - .then(data => { - _assertNoError(null, 'putting second object'); - assert.strictEqual(data.VersionId, undefined); - eTags.push(data.ETag); - callback(); - }) - .catch(callback), - callback => s3.send(new HeadObjectCommand(paramsNull)) - .then(data => { - _assertNoError(null, 'heading null version'); - assert.strictEqual(data.VersionId, 'null'); - assert.strictEqual(data.ETag, eTags[1], - 'wrong object data'); - callback(); - }) - .catch(callback), - ], done); + async.waterfall( + [ + callback => + s3 + .send(new PutObjectCommand(params1)) + .then(data => { + _assertNoError(null, 'putting first object'); + assert.strictEqual(data.VersionId, undefined); + eTags.push(data.ETag); + callback(); + }) + .catch(callback), + callback => + s3 + .send(new HeadObjectCommand(params)) + .then(data => { + _assertNoError(null, 'heading master version'); + assert.strictEqual(data.VersionId, 'null'); + assert.strictEqual(data.ETag, eTags[0], 'wrong object data'); + callback(); + }) + .catch(callback), + callback => + s3 + .send(new PutObjectCommand(params2)) + .then(data => { + _assertNoError(null, 'putting second object'); + assert.strictEqual(data.VersionId, undefined); + eTags.push(data.ETag); + callback(); + }) + .catch(callback), + callback => + s3 + .send(new HeadObjectCommand(paramsNull)) + .then(data => { + _assertNoError(null, 'heading null version'); + assert.strictEqual(data.VersionId, 'null'); + assert.strictEqual(data.ETag, eTags[1], 'wrong object data'); + callback(); + }) + .catch(callback), + ], + done, + ); }); }); - describe('on a version-suspended bucket with non-versioned object', - () => { + describe('on a version-suspended bucket with non-versioned object', () => { const eTags = []; beforeEach(done => { - s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: data[0] - })) + s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: data[0], + }), + ) .then(data => { eTags.push(data.ETag); - return s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningSuspended, - })); + return s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningSuspended, + }), + ); }) .then(() => done()) .catch(done); @@ -318,8 +339,7 @@ describe('put and head object with versioning', function testSuite() { done(); }); - it('should head null version in versioning suspended bucket', - done => { + it('should head null version in versioning suspended bucket', done => { const paramsNull = { Bucket: bucket, Key: '/', @@ -333,8 +353,7 @@ describe('put and head object with versioning', function testSuite() { .catch(done); }); - it('should update null version in versioning suspended bucket', - done => { + it('should update null version in versioning suspended bucket', done => { const params = { Bucket: bucket, Key: key }; const putParams = { Bucket: bucket, Key: '/', Body: data[1] }; const paramsNull = { @@ -342,69 +361,90 @@ describe('put and head object with versioning', function testSuite() { Key: '/', VersionId: 'null', }; - async.waterfall([ - callback => s3.send(new HeadObjectCommand(paramsNull)) - .then(data => { - _assertNoError(null, 'heading null version'); - assert.strictEqual(data.VersionId, 'null'); - callback(); - }) - .catch(callback), - callback => s3.send(new PutObjectCommand(putParams)) - .then(data => { - _assertNoError(null, 'putting object'); - assert.strictEqual(data.VersionId, undefined); - eTags.push(data.ETag); - callback(); - }) - .catch(callback), - callback => s3.send(new HeadObjectCommand(paramsNull)) - .then(data => { - _assertNoError(null, 'heading null version'); - assert.strictEqual(data.VersionId, 'null'); - assert.strictEqual(data.ETag, eTags[1], - 'wrong object data'); - callback(); - }) - .catch(callback), - callback => s3.send(new HeadObjectCommand(params)) - .then(data => { - _assertNoError(null, 'heading master version'); - assert.strictEqual(data.VersionId, 'null'); - assert.strictEqual(data.ETag, eTags[1], - 'wrong object data'); - callback(); - }) - .catch(callback), - ], done); + async.waterfall( + [ + callback => + s3 + .send(new HeadObjectCommand(paramsNull)) + .then(data => { + _assertNoError(null, 'heading null version'); + assert.strictEqual(data.VersionId, 'null'); + callback(); + }) + .catch(callback), + callback => + s3 + .send(new PutObjectCommand(putParams)) + .then(data => { + _assertNoError(null, 'putting object'); + assert.strictEqual(data.VersionId, undefined); + eTags.push(data.ETag); + callback(); + }) + .catch(callback), + callback => + s3 + .send(new HeadObjectCommand(paramsNull)) + .then(data => { + _assertNoError(null, 'heading null version'); + assert.strictEqual(data.VersionId, 'null'); + assert.strictEqual(data.ETag, eTags[1], 'wrong object data'); + callback(); + }) + .catch(callback), + callback => + s3 + .send(new HeadObjectCommand(params)) + .then(data => { + _assertNoError(null, 'heading master version'); + assert.strictEqual(data.VersionId, 'null'); + assert.strictEqual(data.ETag, eTags[1], 'wrong object data'); + callback(); + }) + .catch(callback), + ], + done, + ); }); }); - describe('on versioning suspended then enabled bucket w/ null version', - () => { + describe('on versioning suspended then enabled bucket w/ null version', () => { const eTags = []; beforeEach(done => { const params = { Bucket: bucket, Key: key, Body: data[0] }; - async.waterfall([ - callback => s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningSuspended, - })) - .then(() => callback()) - .catch(callback), - callback => s3.send(new PutObjectCommand(params)) - .then(data => { - eTags.push(data.ETag); - callback(); - }) - .catch(callback), - callback => s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningEnabled, - })) - .then(() => callback()) - .catch(callback), - ], done); + async.waterfall( + [ + callback => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningSuspended, + }), + ) + .then(() => callback()) + .catch(callback), + callback => + s3 + .send(new PutObjectCommand(params)) + .then(data => { + eTags.push(data.ETag); + callback(); + }) + .catch(callback), + callback => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningEnabled, + }), + ) + .then(() => callback()) + .catch(callback), + ], + done, + ); }); afterEach(done => { @@ -413,40 +453,51 @@ describe('put and head object with versioning', function testSuite() { done(); }); - it('should preserve the null version when creating new versions', - done => { + it('should preserve the null version when creating new versions', done => { const params = { Bucket: bucket, Key: key }; const paramsNull = { Bucket: bucket, Key: '/', VersionId: 'null', }; - async.waterfall([ - cb => s3.send(new HeadObjectCommand(paramsNull)) - .then(nullVerData => { - _assertNoError(null, 'heading null version'); - assert.strictEqual(nullVerData.ETag, eTags[0]); - assert.strictEqual(nullVerData.VersionId, 'null'); - cb(); - }) - .catch(cb), - cb => async.timesSeries(counter, (i, next) => - s3.send(new PutObjectCommand(params)) - .then(data => { - _assertNoError(null, `putting object #${i}`); - assert.notEqual(data.VersionId, undefined); - next(); - }) - .catch(next), - err => cb(err)), - cb => s3.send(new HeadObjectCommand(paramsNull)) - .then(nullVerData => { - _assertNoError(null, 'heading null version'); - assert.strictEqual(nullVerData.ETag, eTags[0]); - cb(); - }) - .catch(cb), - ], done); + async.waterfall( + [ + cb => + s3 + .send(new HeadObjectCommand(paramsNull)) + .then(nullVerData => { + _assertNoError(null, 'heading null version'); + assert.strictEqual(nullVerData.ETag, eTags[0]); + assert.strictEqual(nullVerData.VersionId, 'null'); + cb(); + }) + .catch(cb), + cb => + async.timesSeries( + counter, + (i, next) => + s3 + .send(new PutObjectCommand(params)) + .then(data => { + _assertNoError(null, `putting object #${i}`); + assert.notEqual(data.VersionId, undefined); + next(); + }) + .catch(next), + err => cb(err), + ), + cb => + s3 + .send(new HeadObjectCommand(paramsNull)) + .then(nullVerData => { + _assertNoError(null, 'heading null version'); + assert.strictEqual(nullVerData.ETag, eTags[0]); + cb(); + }) + .catch(cb), + ], + done, + ); }); it('should create a bunch of objects and their versions', done => { @@ -454,23 +505,33 @@ describe('put and head object with versioning', function testSuite() { const keycount = 50; const versioncount = 20; const value = '{"foo":"bar"}'; - async.timesLimit(keycount, 10, (i, next1) => { - const key = `foo${i}`; - const params = { Bucket: bucket, Key: key, Body: value }; - async.timesLimit(versioncount, 10, (j, next2) => - s3.send(new PutObjectCommand(params)) - .then(data => { - assert(data.VersionId, 'invalid versionId'); - vids.push({ Key: key, VersionId: data.VersionId }); - next2(); - }) - .catch(next2), - next1); - }, err => { - assert.strictEqual(err, null); - assert.strictEqual(vids.length, keycount * versioncount); - done(); - }); + async.timesLimit( + keycount, + 10, + (i, next1) => { + const key = `foo${i}`; + const params = { Bucket: bucket, Key: key, Body: value }; + async.timesLimit( + versioncount, + 10, + (j, next2) => + s3 + .send(new PutObjectCommand(params)) + .then(data => { + assert(data.VersionId, 'invalid versionId'); + vids.push({ Key: key, VersionId: data.VersionId }); + next2(); + }) + .catch(next2), + next1, + ); + }, + err => { + assert.strictEqual(err, null); + assert.strictEqual(vids.length, keycount * versioncount); + done(); + }, + ); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/versioning/objectPut.js b/tests/functional/aws-node-sdk/test/versioning/objectPut.js index fd61452b3b..f68a12e76b 100644 --- a/tests/functional/aws-node-sdk/test/versioning/objectPut.js +++ b/tests/functional/aws-node-sdk/test/versioning/objectPut.js @@ -75,10 +75,12 @@ describe('put and get object with versioning', function testSuite() { it('should put and get a non-versioned object without including version ids in response headers', async () => { const params = { Bucket: bucket, Key: key, Body: Buffer.from('') }; - const putRes = await s3.send(new PutObjectCommand({ - ...params, - Body: Buffer.from(''), - })); + const putRes = await s3.send( + new PutObjectCommand({ + ...params, + Body: Buffer.from(''), + }), + ); assert.strictEqual(putRes.VersionId, undefined); const getRes = await s3.send(new GetObjectCommand(params)); @@ -87,40 +89,47 @@ describe('put and get object with versioning', function testSuite() { it('version-specific get should still not return version id in response header', async () => { const params = { Bucket: bucket, Key: key, Body: Buffer.from('') }; - const putRes = await s3.send(new PutObjectCommand({ - ...params, - Body: Buffer.from(''), - })); + const putRes = await s3.send( + new PutObjectCommand({ + ...params, + Body: Buffer.from(''), + }), + ); assert.strictEqual(putRes.VersionId, undefined); - const getRes = await s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: 'null', - })); + const getRes = await s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: 'null', + }), + ); assert.strictEqual(getRes.VersionId, undefined); }); describe('on a version-enabled bucket', () => { beforeEach(async () => { - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningEnabled, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningEnabled, + }), + ); }); it('should create a new version for an object', async () => { const params = { Bucket: bucket, Key: key, Body: Buffer.from('') }; const putRes = await s3.send(new PutObjectCommand(params)); - const getRes = await s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: putRes.VersionId, - })); + const getRes = await s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: putRes.VersionId, + }), + ); - assert.strictEqual(putRes.VersionId, getRes.VersionId, - 'version ids are not equal'); + assert.strictEqual(putRes.VersionId, getRes.VersionId, 'version ids are not equal'); }); it('should create a new version with tag set for an object', async () => { @@ -134,14 +143,15 @@ describe('put and get object with versioning', function testSuite() { const putRes = await s3.send(new PutObjectCommand(putParams)); - const tagRes = await s3.send(new GetObjectTaggingCommand({ - Bucket: bucket, - Key: key, - VersionId: putRes.VersionId, - })); + const tagRes = await s3.send( + new GetObjectTaggingCommand({ + Bucket: bucket, + Key: key, + VersionId: putRes.VersionId, + }), + ); - assert.strictEqual(tagRes.VersionId, putRes.VersionId, - 'version ids are not equal'); + assert.strictEqual(tagRes.VersionId, putRes.VersionId, 'version ids are not equal'); assert.strictEqual(tagRes.TagSet[0].Key, tagKey); assert.strictEqual(tagRes.TagSet[0].Value, tagValue); }); @@ -151,68 +161,82 @@ describe('put and get object with versioning', function testSuite() { const eTags = []; beforeEach(async () => { - const putRes = await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: data[0], - })); + const putRes = await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: data[0], + }), + ); eTags.length = 0; eTags.push(putRes.ETag); - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningEnabled, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningEnabled, + }), + ); }); afterEach(() => { eTags.length = 0; }); - it('should get null (latest) version in versioning enabled ' + - 'bucket when version id is not specified', async () => { - const res = await s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - })); - - assert.strictEqual(res.VersionId, 'null'); - }); - - it('should get null version in versioning enabled bucket ' + - 'when version id is specified', async () => { - const res = await s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: 'null', - })); + it( + 'should get null (latest) version in versioning enabled ' + 'bucket when version id is not specified', + async () => { + const res = await s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + }), + ); + + assert.strictEqual(res.VersionId, 'null'); + }, + ); + + it('should get null version in versioning enabled bucket ' + 'when version id is specified', async () => { + const res = await s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: 'null', + }), + ); assert.strictEqual(res.VersionId, 'null'); }); it('should keep null version and create a new version', async () => { - const putRes = await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: data[1], - })); + const putRes = await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: data[1], + }), + ); const newVersion = putRes.VersionId; eTags.push(putRes.ETag); - const newVerRes = await s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: newVersion, - })); - assert.strictEqual(newVerRes.VersionId, newVersion, - 'version ids are not equal'); + const newVerRes = await s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: newVersion, + }), + ); + assert.strictEqual(newVerRes.VersionId, newVersion, 'version ids are not equal'); assert.strictEqual(newVerRes.ETag, eTags[1]); - const nullRes = await s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: 'null', - })); + const nullRes = await s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: 'null', + }), + ); assert.strictEqual(nullRes.VersionId, 'null'); assert.strictEqual(nullRes.ETag, eTags[0]); }); @@ -236,52 +260,67 @@ describe('put and get object with versioning', function testSuite() { }); // S3C-5139 - it('should not fail PUT on versioning-suspended bucket if nullVersionId refers ' + - 'to deleted null version', async () => { - // create a new version on top of non-versioned object - await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - })); - - // suspend versioning - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningSuspended, - })); - - // delete existing non-versioned object - await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: 'null', - })); - - // put a new null version - const putRes = await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: data[0], - })); - eTags[0] = putRes.ETag; - - // get the new null version - const nullVerData = await s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: 'null', - })); - assert.strictEqual(nullVerData.ETag, eTags[0]); - assert.strictEqual(nullVerData.VersionId, 'null'); - }); + it( + 'should not fail PUT on versioning-suspended bucket if nullVersionId refers ' + + 'to deleted null version', + async () => { + // create a new version on top of non-versioned object + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + }), + ); + + // suspend versioning + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningSuspended, + }), + ); + + // delete existing non-versioned object + await s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: 'null', + }), + ); + + // put a new null version + const putRes = await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: data[0], + }), + ); + eTags[0] = putRes.ETag; + + // get the new null version + const nullVerData = await s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: 'null', + }), + ); + assert.strictEqual(nullVerData.ETag, eTags[0]); + assert.strictEqual(nullVerData.VersionId, 'null'); + }, + ); }); describe('on version-suspended bucket', () => { beforeEach(async () => { - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningSuspended, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningSuspended, + }), + ); }); it('should not return version id for new object', async () => { @@ -334,86 +373,103 @@ describe('put and get object with versioning', function testSuite() { }); // Jira issue: S3C-444 - it('put object after put object acl on null version which is ' + - 'latest version should not result in two null version with ' + - 'different version ids', async () => { - // create new null version (master version in metadata) - await s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: Buffer.from('') })); - await checkOneVersion(s3, bucket, 'null'); - - // apply ACL on null version - await s3.send(new PutObjectAclCommand({ - Bucket: bucket, - Key: key, - ACL: 'public-read-write', - VersionId: 'null', - })); - - // before overwriting master version, put object should clean up latest null version - await s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: Buffer.from('') })); - - // if clean-up did not occur, would see two null versions with different version IDs - await checkOneVersion(s3, bucket, 'null'); - }); + it( + 'put object after put object acl on null version which is ' + + 'latest version should not result in two null version with ' + + 'different version ids', + async () => { + // create new null version (master version in metadata) + await s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: Buffer.from('') })); + await checkOneVersion(s3, bucket, 'null'); + + // apply ACL on null version + await s3.send( + new PutObjectAclCommand({ + Bucket: bucket, + Key: key, + ACL: 'public-read-write', + VersionId: 'null', + }), + ); + + // before overwriting master version, put object should clean up latest null version + await s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: Buffer.from('') })); + + // if clean-up did not occur, would see two null versions with different version IDs + await checkOneVersion(s3, bucket, 'null'); + }, + ); // Jira issue: S3C-444 - it('put object after creating dual null version another way ' + - 'should not result in two null version with different version ids', async () => { - // create dual null version state another way - await createDualNullVersionAsync(s3, bucket, key); - - // versioning is left enabled after above step - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningSuspended, - })); - - // before overwriting master version, put object should clean up latest null version - await s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: Buffer.from('') })); - - // if clean-up did not occur, would see two null versions with different version IDs - await checkOneVersion(s3, bucket, 'null'); - }); + it( + 'put object after creating dual null version another way ' + + 'should not result in two null version with different version ids', + async () => { + // create dual null version state another way + await createDualNullVersionAsync(s3, bucket, key); + + // versioning is left enabled after above step + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningSuspended, + }), + ); + + // before overwriting master version, put object should clean up latest null version + await s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: Buffer.from('') })); + + // if clean-up did not occur, would see two null versions with different version IDs + await checkOneVersion(s3, bucket, 'null'); + }, + ); }); describe('on a version-suspended bucket with non-versioned object', () => { const eTags = []; beforeEach(async () => { - const putRes = await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: data[0], - })); + const putRes = await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: data[0], + }), + ); eTags.length = 0; eTags.push(putRes.ETag); - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningSuspended, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningSuspended, + }), + ); }); afterEach(() => { eTags.length = 0; }); - it('should get null version (latest) in versioning suspended bucket without specifying version id', - async () => { - const res = await s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - })); + it('should get null version (latest) in versioning suspended bucket without specifying version id', async () => { + const res = await s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + }), + ); assert.strictEqual(res.VersionId, 'null'); }); it('should get null version in versioning suspended bucket specifying version id', async () => { - const res = await s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: 'null', - })); + const res = await s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: 'null', + }), + ); assert.strictEqual(res.VersionId, 'null'); }); @@ -450,19 +506,23 @@ describe('put and get object with versioning', function testSuite() { beforeEach(async () => { const params = { Bucket: bucket, Key: key, Body: data[0] }; - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningSuspended, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningSuspended, + }), + ); const putRes = await s3.send(new PutObjectCommand(params)); eTags.length = 0; eTags.push(putRes.ETag); - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningEnabled, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningEnabled, + }), + ); }); afterEach(() => { diff --git a/tests/functional/aws-node-sdk/test/versioning/objectPutCopyPart.js b/tests/functional/aws-node-sdk/test/versioning/objectPutCopyPart.js index fa1183b2cf..329bf0f894 100644 --- a/tests/functional/aws-node-sdk/test/versioning/objectPutCopyPart.js +++ b/tests/functional/aws-node-sdk/test/versioning/objectPutCopyPart.js @@ -17,11 +17,7 @@ const { const withV4 = require('../support/withV4'); const BucketUtility = require('../../lib/utility/bucket-util'); -const { - removeAllVersions, - versioningEnabled, - versioningSuspended, -} = require('../../lib/utility/versioning-util.js'); +const { removeAllVersions, versioningEnabled, versioningSuspended } = require('../../lib/utility/versioning-util.js'); const removeAllVersionsPromise = promisify(removeAllVersions); let sourceBucket; @@ -30,7 +26,6 @@ const sourceKey = 'sourceobjectkey'; const destKey = 'destobjectkey'; const invalidId = 'invalidIdWithMoreThan40BytesAndThatIsNotLongEnoughYet'; - describe('Object Part Copy with Versioning', () => { withV4(sigCfg => { const bucketUtil = new BucketUtility('default', sigCfg); @@ -40,26 +35,36 @@ describe('Object Part Copy with Versioning', () => { beforeEach(done => { sourceBucket = `copypartsourcebucket-${Date.now()}`; destBucket = `copypartdestbucket-${Date.now()}`; - async.forEach([sourceBucket, destBucket], (bucket, cb) => { - s3.send(new CreateBucketCommand({ Bucket: bucket })) - .then(() => cb()) - .catch(cb); - }, done); + async.forEach( + [sourceBucket, destBucket], + (bucket, cb) => { + s3.send(new CreateBucketCommand({ Bucket: bucket })) + .then(() => cb()) + .catch(cb); + }, + done, + ); }); afterEach(done => { - s3.send(new AbortMultipartUploadCommand({ - Bucket: destBucket, - Key: destKey, - UploadId: uploadId, - })) + s3.send( + new AbortMultipartUploadCommand({ + Bucket: destBucket, + Key: destKey, + UploadId: uploadId, + }), + ) .then(() => { - async.each([sourceBucket, destBucket], (bucket, cb) => { - removeAllVersionsPromise({ Bucket: bucket }) - .then(() => s3.send(new DeleteBucketCommand({ Bucket: bucket }))) - .then(() => cb()) - .catch(cb); - }, done); + async.each( + [sourceBucket, destBucket], + (bucket, cb) => { + removeAllVersionsPromise({ Bucket: bucket }) + .then(() => s3.send(new DeleteBucketCommand({ Bucket: bucket }))) + .then(() => cb()) + .catch(cb); + }, + done, + ); }) .catch(err => { if (err) { @@ -73,30 +78,39 @@ describe('Object Part Copy with Versioning', () => { const eTags = []; beforeEach(done => { - async.waterfall([ - next => s3.send(new PutObjectCommand({ - Bucket: sourceBucket, - Key: sourceKey, - Body: 'foobar' - })) - .then(data => next(null, data)) - .catch(next), - (data, next) => { - eTags.push(data.ETag); - s3.send(new CreateMultipartUploadCommand({ - Bucket: destBucket, - Key: destKey - })) - .then(data => next(null, data)) - .catch(next); + async.waterfall( + [ + next => + s3 + .send( + new PutObjectCommand({ + Bucket: sourceBucket, + Key: sourceKey, + Body: 'foobar', + }), + ) + .then(data => next(null, data)) + .catch(next), + (data, next) => { + eTags.push(data.ETag); + s3.send( + new CreateMultipartUploadCommand({ + Bucket: destBucket, + Key: destKey, + }), + ) + .then(data => next(null, data)) + .catch(next); + }, + ], + (err, data) => { + if (err) { + return done(err); + } + uploadId = data.UploadId; + return done(); }, - ], (err, data) => { - if (err) { - return done(err); - } - uploadId = data.UploadId; - return done(); - }); + ); }); afterEach(done => { @@ -104,15 +118,16 @@ describe('Object Part Copy with Versioning', () => { done(); }); - it('should not return a version id when put part by copying ' + - 'without specifying version id', done => { - s3.send(new UploadPartCopyCommand({ - Bucket: destBucket, - CopySource: `${sourceBucket}/${sourceKey}`, - Key: destKey, - PartNumber: 1, - UploadId: uploadId, - })) + it('should not return a version id when put part by copying ' + 'without specifying version id', done => { + s3.send( + new UploadPartCopyCommand({ + Bucket: destBucket, + CopySource: `${sourceBucket}/${sourceKey}`, + Key: destKey, + PartNumber: 1, + UploadId: uploadId, + }), + ) .then(data => { assert.strictEqual(data.CopySourceVersionId, undefined); assert.strictEqual(data.CopyPartResult.ETag, eTags[0]); @@ -121,16 +136,16 @@ describe('Object Part Copy with Versioning', () => { .catch(done); }); - it('should return NoSuchKey if copy source version id is invalid ' + - 'id', done => { - s3.send(new UploadPartCopyCommand({ - Bucket: destBucket, - CopySource: `${sourceBucket}/${sourceKey}?` + - `versionId=${invalidId}`, - Key: destKey, - PartNumber: 1, - UploadId: uploadId, - })) + it('should return NoSuchKey if copy source version id is invalid ' + 'id', done => { + s3.send( + new UploadPartCopyCommand({ + Bucket: destBucket, + CopySource: `${sourceBucket}/${sourceKey}?` + `versionId=${invalidId}`, + Key: destKey, + PartNumber: 1, + UploadId: uploadId, + }), + ) .then(() => { done(new Error('Expected error but got success')); }) @@ -142,22 +157,27 @@ describe('Object Part Copy with Versioning', () => { }); }); - it('should allow specific version "null" for copy source ' + - 'and return version id "null" in response headers', done => { - s3.send(new UploadPartCopyCommand({ - Bucket: destBucket, - CopySource: `${sourceBucket}/${sourceKey}?versionId=null`, - Key: destKey, - PartNumber: 1, - UploadId: uploadId, - })) - .then(data => { - assert.strictEqual(data.CopySourceVersionId, 'null'); - assert.strictEqual(data.CopyPartResult.ETag, eTags[0]); - done(); - }) - .catch(done); - }); + it( + 'should allow specific version "null" for copy source ' + + 'and return version id "null" in response headers', + done => { + s3.send( + new UploadPartCopyCommand({ + Bucket: destBucket, + CopySource: `${sourceBucket}/${sourceKey}?versionId=null`, + Key: destKey, + PartNumber: 1, + UploadId: uploadId, + }), + ) + .then(data => { + assert.strictEqual(data.CopySourceVersionId, 'null'); + assert.strictEqual(data.CopyPartResult.ETag, eTags[0]); + done(); + }) + .catch(done); + }, + ); }); describe('on bucket with versioning', () => { @@ -167,46 +187,64 @@ describe('Object Part Copy with Versioning', () => { beforeEach(done => { const params = { Bucket: sourceBucket, Key: sourceKey }; - async.waterfall([ - next => s3.send(new PutObjectCommand(params)) - .then(data => next(null, data)) - .catch(next), - (data, next) => { - eTags.push(data.ETag); - versionIds.push('null'); - s3.send(new PutBucketVersioningCommand({ - Bucket: sourceBucket, - VersioningConfiguration: versioningEnabled, - })) - .then(() => next()) - .catch(next); + async.waterfall( + [ + next => + s3 + .send(new PutObjectCommand(params)) + .then(data => next(null, data)) + .catch(next), + (data, next) => { + eTags.push(data.ETag); + versionIds.push('null'); + s3.send( + new PutBucketVersioningCommand({ + Bucket: sourceBucket, + VersioningConfiguration: versioningEnabled, + }), + ) + .then(() => next()) + .catch(next); + }, + next => + async.timesSeries( + counter, + (i, cb) => + s3 + .send( + new PutObjectCommand({ + Bucket: sourceBucket, + Key: sourceKey, + Body: `foo${i}`, + }), + ) + .then(data => { + eTags.push(data.ETag); + versionIds.push(data.VersionId); + cb(); + }) + .catch(cb), + err => next(err), + ), + next => + s3 + .send( + new CreateMultipartUploadCommand({ + Bucket: destBucket, + Key: destKey, + }), + ) + .then(data => next(null, data)) + .catch(next), + ], + (err, data) => { + if (err) { + return done(err); + } + uploadId = data.UploadId; + return done(); }, - next => async.timesSeries(counter, (i, cb) => - s3.send(new PutObjectCommand({ - Bucket: sourceBucket, - Key: sourceKey, - Body: `foo${i}` - })) - .then(data => { - eTags.push(data.ETag); - versionIds.push(data.VersionId); - cb(); - }) - .catch(cb), - err => next(err)), - next => s3.send(new CreateMultipartUploadCommand({ - Bucket: destBucket, - Key: destKey - })) - .then(data => next(null, data)) - .catch(next), - ], (err, data) => { - if (err) { - return done(err); - } - uploadId = data.UploadId; - return done(); - }); + ); }); afterEach(done => { @@ -215,102 +253,132 @@ describe('Object Part Copy with Versioning', () => { done(); }); - it('copy part without specifying version should return data and ' + - 'version id of latest version', done => { - const lastVersion = versionIds[versionIds.length - 1]; - const lastETag = eTags[eTags.length - 1]; - s3.send(new UploadPartCopyCommand({ - Bucket: destBucket, - CopySource: `${sourceBucket}/${sourceKey}`, - Key: destKey, - PartNumber: 1, - UploadId: uploadId, - })) - .then(data => { - assert.strictEqual(data.CopySourceVersionId, lastVersion); - assert.strictEqual(data.CopyPartResult.ETag, lastETag); - done(); - }) - .catch(done); - }); - - it('copy part without specifying version should return NoSuchKey ' + - 'if latest version has a delete marker', done => { - s3.send(new DeleteObjectCommand({ - Bucket: sourceBucket, - Key: sourceKey - })) - .then(() => s3.send(new UploadPartCopyCommand({ + it( + 'copy part without specifying version should return data and ' + 'version id of latest version', + done => { + const lastVersion = versionIds[versionIds.length - 1]; + const lastETag = eTags[eTags.length - 1]; + s3.send( + new UploadPartCopyCommand({ Bucket: destBucket, CopySource: `${sourceBucket}/${sourceKey}`, Key: destKey, PartNumber: 1, UploadId: uploadId, - }))) - .then(() => { - done(new Error('Expected err but did not find one')); - }) - .catch(err => { - assert(err, 'Expected err but did not find one'); - assert.strictEqual(err.name, 'NoSuchKey'); - assert.strictEqual(err.$metadata?.httpStatusCode, 404); - done(); - }); - }); + }), + ) + .then(data => { + assert.strictEqual(data.CopySourceVersionId, lastVersion); + assert.strictEqual(data.CopyPartResult.ETag, lastETag); + done(); + }) + .catch(done); + }, + ); - it('copy part with specific version id should return ' + - 'InvalidRequest if that id is a delete marker', done => { - async.waterfall([ - next => s3.send(new DeleteObjectCommand({ - Bucket: sourceBucket, - Key: sourceKey, - })) - .then(() => next()) - .catch(next), - next => s3.send(new ListObjectVersionsCommand({ - Bucket: sourceBucket - })) - .then(data => next(null, data)) - .catch(next), - (data, next) => { - const deleteMarkerId = data.DeleteMarkers[0].VersionId; - return s3.send(new UploadPartCopyCommand({ - Bucket: destBucket, - CopySource: `${sourceBucket}/${sourceKey}` + - `?versionId=${deleteMarkerId}`, - Key: destKey, - PartNumber: 1, - UploadId: uploadId, - })) - .then(data => next(null, data)) - .catch(next); - }, - ], err => { - assert(err, 'Expected err but did not find one'); - assert.strictEqual(err.name, 'InvalidRequest'); - assert.strictEqual(err.$metadata?.httpStatusCode, 400); - done(); - }); - }); + it( + 'copy part without specifying version should return NoSuchKey ' + + 'if latest version has a delete marker', + done => { + s3.send( + new DeleteObjectCommand({ + Bucket: sourceBucket, + Key: sourceKey, + }), + ) + .then(() => + s3.send( + new UploadPartCopyCommand({ + Bucket: destBucket, + CopySource: `${sourceBucket}/${sourceKey}`, + Key: destKey, + PartNumber: 1, + UploadId: uploadId, + }), + ), + ) + .then(() => { + done(new Error('Expected err but did not find one')); + }) + .catch(err => { + assert(err, 'Expected err but did not find one'); + assert.strictEqual(err.name, 'NoSuchKey'); + assert.strictEqual(err.$metadata?.httpStatusCode, 404); + done(); + }); + }, + ); + + it( + 'copy part with specific version id should return ' + 'InvalidRequest if that id is a delete marker', + done => { + async.waterfall( + [ + next => + s3 + .send( + new DeleteObjectCommand({ + Bucket: sourceBucket, + Key: sourceKey, + }), + ) + .then(() => next()) + .catch(next), + next => + s3 + .send( + new ListObjectVersionsCommand({ + Bucket: sourceBucket, + }), + ) + .then(data => next(null, data)) + .catch(next), + (data, next) => { + const deleteMarkerId = data.DeleteMarkers[0].VersionId; + return s3 + .send( + new UploadPartCopyCommand({ + Bucket: destBucket, + CopySource: `${sourceBucket}/${sourceKey}` + `?versionId=${deleteMarkerId}`, + Key: destKey, + PartNumber: 1, + UploadId: uploadId, + }), + ) + .then(data => next(null, data)) + .catch(next); + }, + ], + err => { + assert(err, 'Expected err but did not find one'); + assert.strictEqual(err.name, 'InvalidRequest'); + assert.strictEqual(err.$metadata?.httpStatusCode, 400); + done(); + }, + ); + }, + ); - it('copy part with specific version should return NoSuchVersion ' + - 'if version does not exist', done => { + it('copy part with specific version should return NoSuchVersion ' + 'if version does not exist', done => { const versionId = versionIds[1]; - s3.send(new DeleteObjectCommand({ - Bucket: sourceBucket, - Key: sourceKey, - VersionId: versionId - })) + s3.send( + new DeleteObjectCommand({ + Bucket: sourceBucket, + Key: sourceKey, + VersionId: versionId, + }), + ) .then(data => { assert.strictEqual(data.VersionId, versionId); - return s3.send(new UploadPartCopyCommand({ - Bucket: destBucket, - CopySource: `${sourceBucket}/${sourceKey}` + - `?versionId=${versionId}`, - Key: destKey, - PartNumber: 1, - UploadId: uploadId, - })); + return s3.send( + new UploadPartCopyCommand({ + Bucket: destBucket, + CopySource: `${sourceBucket}/${sourceKey}` + `?versionId=${versionId}`, + Key: destKey, + PartNumber: 1, + UploadId: uploadId, + }), + ); }) .then(() => { done(new Error('Expected err but did not find one')); @@ -323,17 +391,17 @@ describe('Object Part Copy with Versioning', () => { }); }); - it('copy part with specific version should return copy source ' + - 'version id if it exists', done => { + it('copy part with specific version should return copy source ' + 'version id if it exists', done => { const versionId = versionIds[1]; - s3.send(new UploadPartCopyCommand({ - Bucket: destBucket, - CopySource: `${sourceBucket}/${sourceKey}` + - `?versionId=${versionId}`, - Key: destKey, - PartNumber: 1, - UploadId: uploadId, - })) + s3.send( + new UploadPartCopyCommand({ + Bucket: destBucket, + CopySource: `${sourceBucket}/${sourceKey}` + `?versionId=${versionId}`, + Key: destKey, + PartNumber: 1, + UploadId: uploadId, + }), + ) .then(data => { assert.strictEqual(data.CopySourceVersionId, versionId); assert.strictEqual(data.CopyPartResult.ETag, eTags[1]); @@ -342,22 +410,26 @@ describe('Object Part Copy with Versioning', () => { .catch(done); }); - it('copy part with specific version "null" should return copy ' + - 'source version id "null" if it exists', done => { - s3.send(new UploadPartCopyCommand({ - Bucket: destBucket, - CopySource: `${sourceBucket}/${sourceKey}?versionId=null`, - Key: destKey, - PartNumber: 1, - UploadId: uploadId, - })) - .then(data => { - assert.strictEqual(data.CopySourceVersionId, 'null'); - assert.strictEqual(data.CopyPartResult.ETag, eTags[0]); - done(); - }) - .catch(done); - }); + it( + 'copy part with specific version "null" should return copy ' + 'source version id "null" if it exists', + done => { + s3.send( + new UploadPartCopyCommand({ + Bucket: destBucket, + CopySource: `${sourceBucket}/${sourceKey}?versionId=null`, + Key: destKey, + PartNumber: 1, + UploadId: uploadId, + }), + ) + .then(data => { + assert.strictEqual(data.CopySourceVersionId, 'null'); + assert.strictEqual(data.CopyPartResult.ETag, eTags[0]); + done(); + }) + .catch(done); + }, + ); }); describe('on bucket with versioning suspended', () => { @@ -367,54 +439,74 @@ describe('Object Part Copy with Versioning', () => { beforeEach(done => { const params = { Bucket: sourceBucket, Key: sourceKey }; - async.waterfall([ - next => s3.send(new PutObjectCommand(params)) - .then(data => next(null, data)) - .catch(next), - (data, next) => { - eTags.push(data.ETag); - versionIds.push('null'); - s3.send(new PutBucketVersioningCommand({ - Bucket: sourceBucket, - VersioningConfiguration: versioningEnabled, - })) - .then(() => next()) - .catch(next); - }, - next => async.timesSeries(counter, (i, cb) => - s3.send(new PutObjectCommand({ - Bucket: sourceBucket, - Key: sourceKey, - Body: `foo${i}` - })) - .then(data => { - eTags.push(data.ETag); - versionIds.push(data.VersionId); - cb(); - }) - .catch(cb), - err => next(err)), - next => { - s3.send(new PutBucketVersioningCommand({ - Bucket: sourceBucket, - VersioningConfiguration: versioningSuspended, - })) - .then(() => next()) - .catch(next); + async.waterfall( + [ + next => + s3 + .send(new PutObjectCommand(params)) + .then(data => next(null, data)) + .catch(next), + (data, next) => { + eTags.push(data.ETag); + versionIds.push('null'); + s3.send( + new PutBucketVersioningCommand({ + Bucket: sourceBucket, + VersioningConfiguration: versioningEnabled, + }), + ) + .then(() => next()) + .catch(next); + }, + next => + async.timesSeries( + counter, + (i, cb) => + s3 + .send( + new PutObjectCommand({ + Bucket: sourceBucket, + Key: sourceKey, + Body: `foo${i}`, + }), + ) + .then(data => { + eTags.push(data.ETag); + versionIds.push(data.VersionId); + cb(); + }) + .catch(cb), + err => next(err), + ), + next => { + s3.send( + new PutBucketVersioningCommand({ + Bucket: sourceBucket, + VersioningConfiguration: versioningSuspended, + }), + ) + .then(() => next()) + .catch(next); + }, + next => + s3 + .send( + new CreateMultipartUploadCommand({ + Bucket: destBucket, + Key: destKey, + }), + ) + .then(data => next(null, data)) + .catch(next), + ], + (err, data) => { + if (err) { + return done(err); + } + uploadId = data.UploadId; + return done(); }, - next => s3.send(new CreateMultipartUploadCommand({ - Bucket: destBucket, - Key: destKey - })) - .then(data => next(null, data)) - .catch(next), - ], (err, data) => { - if (err) { - return done(err); - } - uploadId = data.UploadId; - return done(); - }); + ); }); afterEach(done => { @@ -423,17 +515,18 @@ describe('Object Part Copy with Versioning', () => { done(); }); - it('copy part without specifying version should still return ' + - 'version id of latest version', done => { + it('copy part without specifying version should still return ' + 'version id of latest version', done => { const lastVersion = versionIds[versionIds.length - 1]; const lastETag = eTags[eTags.length - 1]; - s3.send(new UploadPartCopyCommand({ - Bucket: destBucket, - CopySource: `${sourceBucket}/${sourceKey}`, - Key: destKey, - PartNumber: 1, - UploadId: uploadId, - })) + s3.send( + new UploadPartCopyCommand({ + Bucket: destBucket, + CopySource: `${sourceBucket}/${sourceKey}`, + Key: destKey, + PartNumber: 1, + UploadId: uploadId, + }), + ) .then(data => { assert.strictEqual(data.CopySourceVersionId, lastVersion); assert.strictEqual(data.CopyPartResult.ETag, lastETag); @@ -442,17 +535,17 @@ describe('Object Part Copy with Versioning', () => { .catch(done); }); - it('copy part with specific version should still return copy ' + - 'source version id if it exists', done => { + it('copy part with specific version should still return copy ' + 'source version id if it exists', done => { const versionId = versionIds[1]; - s3.send(new UploadPartCopyCommand({ - Bucket: destBucket, - CopySource: `${sourceBucket}/${sourceKey}` + - `?versionId=${versionId}`, - Key: destKey, - PartNumber: 1, - UploadId: uploadId, - })) + s3.send( + new UploadPartCopyCommand({ + Bucket: destBucket, + CopySource: `${sourceBucket}/${sourceKey}` + `?versionId=${versionId}`, + Key: destKey, + PartNumber: 1, + UploadId: uploadId, + }), + ) .then(data => { assert.strictEqual(data.CopySourceVersionId, versionId); assert.strictEqual(data.CopyPartResult.ETag, eTags[1]); @@ -461,22 +554,27 @@ describe('Object Part Copy with Versioning', () => { .catch(done); }); - it('copy part with specific version "null" should still return ' + - 'copy source version id "null" if it exists', done => { - s3.send(new UploadPartCopyCommand({ - Bucket: destBucket, - CopySource: `${sourceBucket}/${sourceKey}?versionId=null`, - Key: destKey, - PartNumber: 1, - UploadId: uploadId, - })) - .then(data => { - assert.strictEqual(data.CopySourceVersionId, 'null'); - assert.strictEqual(data.CopyPartResult.ETag, eTags[0]); - done(); - }) - .catch(done); - }); + it( + 'copy part with specific version "null" should still return ' + + 'copy source version id "null" if it exists', + done => { + s3.send( + new UploadPartCopyCommand({ + Bucket: destBucket, + CopySource: `${sourceBucket}/${sourceKey}?versionId=null`, + Key: destKey, + PartNumber: 1, + UploadId: uploadId, + }), + ) + .then(data => { + assert.strictEqual(data.CopySourceVersionId, 'null'); + assert.strictEqual(data.CopyPartResult.ETag, eTags[0]); + done(); + }) + .catch(done); + }, + ); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/versioning/objectPutTagging.js b/tests/functional/aws-node-sdk/test/versioning/objectPutTagging.js index b71919fba8..d2c8fca202 100644 --- a/tests/functional/aws-node-sdk/test/versioning/objectPutTagging.js +++ b/tests/functional/aws-node-sdk/test/versioning/objectPutTagging.js @@ -1,6 +1,6 @@ const assert = require('assert'); const async = require('async'); -const {promisify} = require('util'); +const { promisify } = require('util'); const { CreateBucketCommand, @@ -15,12 +15,9 @@ const withV4 = require('../support/withV4'); const BucketUtility = require('../../lib/utility/bucket-util'); const { checkOneVersion } = require('../../lib/utility/versioning-util'); -const { - removeAllVersions, - versioningEnabled, -} = require('../../lib/utility/versioning-util'); +const { removeAllVersions, versioningEnabled } = require('../../lib/utility/versioning-util'); -const removeAllVersionsPromise= promisify(removeAllVersions); +const removeAllVersionsPromise = promisify(removeAllVersions); const bucketName = 'testtaggingbucket'; const objectName = 'testtaggingobject'; @@ -49,227 +46,351 @@ describe('Put object tagging with versioning', () => { }); it('should be able to put tag with versioning', done => { - async.waterfall([ - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: versioningEnabled, - })) - .then(() => next()) - .catch(next), - - next => s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName - })) - .then(data => next(null, data.VersionId)) - .catch(next), - - (versionId, next) => s3.send(new PutObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - VersionId: versionId, - Tagging: { TagSet: [ - { - Key: 'key1', - Value: 'value1', - }] }, - })) - .then(data => next(null, data, versionId)) - .catch(next), - ], (err, data, versionId) => { - assert.ifError(err, `Found unexpected err ${err}`); - assert.strictEqual(data.VersionId, versionId); - done(); - }); - }); + async.waterfall( + [ + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: versioningEnabled, + }), + ) + .then(() => next()) + .catch(next), + + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ) + .then(data => next(null, data.VersionId)) + .catch(next), - it('should not create version putting object tags on a ' + - ' version-enabled bucket where no version id is specified ', done => { - async.waterfall([ - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: versioningEnabled, - })) - .then(() => next()) - .catch(next), - - next => s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName - })) - .then(data => next(null, data.VersionId)) - .catch(next), - - (versionId, next) => s3.send(new PutObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - Tagging: { TagSet: [ - { - Key: 'key1', - Value: 'value1', - }] }, - })) - .then(() => next(null, versionId)) - .catch(next), - - (versionId, next) => - checkOneVersion(s3, bucketName, versionId) - .then(() => next()) - .catch(next), - ], done); + (versionId, next) => + s3 + .send( + new PutObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + VersionId: versionId, + Tagging: { + TagSet: [ + { + Key: 'key1', + Value: 'value1', + }, + ], + }, + }), + ) + .then(data => next(null, data, versionId)) + .catch(next), + ], + (err, data, versionId) => { + assert.ifError(err, `Found unexpected err ${err}`); + assert.strictEqual(data.VersionId, versionId); + done(); + }, + ); }); + it( + 'should not create version putting object tags on a ' + + ' version-enabled bucket where no version id is specified ', + done => { + async.waterfall( + [ + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: versioningEnabled, + }), + ) + .then(() => next()) + .catch(next), + + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ) + .then(data => next(null, data.VersionId)) + .catch(next), + + (versionId, next) => + s3 + .send( + new PutObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + Tagging: { + TagSet: [ + { + Key: 'key1', + Value: 'value1', + }, + ], + }, + }), + ) + .then(() => next(null, versionId)) + .catch(next), + + (versionId, next) => + checkOneVersion(s3, bucketName, versionId) + .then(() => next()) + .catch(next), + ], + done, + ); + }, + ); + it('should be able to put tag with a version of id "null"', done => { - async.waterfall([ - next => s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName - })) - .then(() => next()) - .catch(next), - - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: versioningEnabled, - })) - .then(() => next()) - .catch(next), - - next => s3.send(new PutObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - VersionId: 'null', - Tagging: { TagSet: [ - { - Key: 'key1', - Value: 'value1', - }] }, - })) - .then(data => next(null, data)) - .catch(next), - ], (err, data) => { - assert.ifError(err, `Found unexpected err ${err}`); - assert.strictEqual(data.VersionId, 'null'); - done(); - }); - }); + async.waterfall( + [ + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ) + .then(() => next()) + .catch(next), - it('should return InvalidArgument putting tag with a non existing ' + - 'version id', done => { - async.waterfall([ - next => s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName - })) - .then(() => next()) - .catch(next), - - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: versioningEnabled, - })) - .then(() => next()) - .catch(next), - - next => s3.send(new PutObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - VersionId: invalidId, - Tagging: { TagSet: [ - { - Key: 'key1', - Value: 'value1', - }] }, - })) - .then(data => next(null, data)) - .catch(next), - ], err => { - _checkError(err, 'InvalidArgument', 400); - done(); - }); - }); + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: versioningEnabled, + }), + ) + .then(() => next()) + .catch(next), - it('should return 405 MethodNotAllowed putting tag without ' + - 'version id if version specified is a delete marker', done => { - async.waterfall([ - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: versioningEnabled, - })) - .then(() => next()) - .catch(next), - - next => s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName - })) - .then(() => next()) - .catch(next), - - next => s3.send(new DeleteObjectCommand({ - Bucket: bucketName, - Key: objectName - })) - .then(() => next()) - .catch(next), - - next => s3.send(new PutObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - Tagging: { TagSet: [ - { - Key: 'key1', - Value: 'value1', - }] }, - })) - .then(data => next(null, data)) - .catch(next), - ], err => { - _checkError(err, 'MethodNotAllowed', 405); - done(); - }); + next => + s3 + .send( + new PutObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + VersionId: 'null', + Tagging: { + TagSet: [ + { + Key: 'key1', + Value: 'value1', + }, + ], + }, + }), + ) + .then(data => next(null, data)) + .catch(next), + ], + (err, data) => { + assert.ifError(err, `Found unexpected err ${err}`); + assert.strictEqual(data.VersionId, 'null'); + done(); + }, + ); }); - it('should return 405 MethodNotAllowed putting tag with ' + - 'version id if version specified is a delete marker', done => { - async.waterfall([ - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: versioningEnabled, - })) - .then(() => next()) - .catch(next), - - next => s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName - })) - .then(() => next()) - .catch(next), - - next => s3.send(new DeleteObjectCommand({ - Bucket: bucketName, - Key: objectName - })) - .then(data => next(null, data.VersionId)) - .catch(next), - - (versionId, next) => s3.send(new PutObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - VersionId: versionId, - Tagging: { TagSet: [ - { - Key: 'key1', - Value: 'value1', - }] }, - })) - .then(data => next(null, data)) - .catch(next), - ], err => { - _checkError(err, 'MethodNotAllowed', 405); - done(); - }); + it('should return InvalidArgument putting tag with a non existing ' + 'version id', done => { + async.waterfall( + [ + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ) + .then(() => next()) + .catch(next), + + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: versioningEnabled, + }), + ) + .then(() => next()) + .catch(next), + + next => + s3 + .send( + new PutObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + VersionId: invalidId, + Tagging: { + TagSet: [ + { + Key: 'key1', + Value: 'value1', + }, + ], + }, + }), + ) + .then(data => next(null, data)) + .catch(next), + ], + err => { + _checkError(err, 'InvalidArgument', 400); + done(); + }, + ); }); + + it( + 'should return 405 MethodNotAllowed putting tag without ' + + 'version id if version specified is a delete marker', + done => { + async.waterfall( + [ + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: versioningEnabled, + }), + ) + .then(() => next()) + .catch(next), + + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ) + .then(() => next()) + .catch(next), + + next => + s3 + .send( + new DeleteObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ) + .then(() => next()) + .catch(next), + + next => + s3 + .send( + new PutObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + Tagging: { + TagSet: [ + { + Key: 'key1', + Value: 'value1', + }, + ], + }, + }), + ) + .then(data => next(null, data)) + .catch(next), + ], + err => { + _checkError(err, 'MethodNotAllowed', 405); + done(); + }, + ); + }, + ); + + it( + 'should return 405 MethodNotAllowed putting tag with ' + + 'version id if version specified is a delete marker', + done => { + async.waterfall( + [ + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: versioningEnabled, + }), + ) + .then(() => next()) + .catch(next), + + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ) + .then(() => next()) + .catch(next), + + next => + s3 + .send( + new DeleteObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ) + .then(data => next(null, data.VersionId)) + .catch(next), + + (versionId, next) => + s3 + .send( + new PutObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + VersionId: versionId, + Tagging: { + TagSet: [ + { + Key: 'key1', + Value: 'value1', + }, + ], + }, + }), + ) + .then(data => next(null, data)) + .catch(next), + ], + err => { + _checkError(err, 'MethodNotAllowed', 405); + done(); + }, + ); + }, + ); }); }); diff --git a/tests/functional/aws-node-sdk/test/versioning/replicationBucket.js b/tests/functional/aws-node-sdk/test/versioning/replicationBucket.js index 9bc9af2ad4..492aaa0f8f 100644 --- a/tests/functional/aws-node-sdk/test/versioning/replicationBucket.js +++ b/tests/functional/aws-node-sdk/test/versioning/replicationBucket.js @@ -23,15 +23,14 @@ function checkNoError(err) { } function testVersioning(s3, versioningStatus, replicationStatus, removeReplication, cb) { - const versioningParams = { + const versioningParams = { Bucket: bucketName, - VersioningConfiguration: { Status: versioningStatus } + VersioningConfiguration: { Status: versioningStatus }, }; const replicationParams = { Bucket: bucketName, ReplicationConfiguration: { - Role: 'arn:aws:iam::123456789012:role/examplerole,' + - 'arn:aws:iam::123456789012:role/examplerole', + Role: 'arn:aws:iam::123456789012:role/examplerole,' + 'arn:aws:iam::123456789012:role/examplerole', Rules: [ { Destination: { @@ -44,23 +43,31 @@ function testVersioning(s3, versioningStatus, replicationStatus, removeReplicati ], }, }; - - async.waterfall([ - cb => s3.send(new PutBucketReplicationCommand(replicationParams)) - .then(() => cb()) - .catch(cb), - cb => { - if (removeReplication) { - return s3.send(new DeleteBucketReplicationCommand({ Bucket: bucketName })) + + async.waterfall( + [ + cb => + s3 + .send(new PutBucketReplicationCommand(replicationParams)) .then(() => cb()) - .catch(cb); - } - return process.nextTick(() => cb()); - }, - cb => s3.send(new PutBucketVersioningCommand(versioningParams)) - .then(() => cb()) - .catch(cb), - ], cb); + .catch(cb), + cb => { + if (removeReplication) { + return s3 + .send(new DeleteBucketReplicationCommand({ Bucket: bucketName })) + .then(() => cb()) + .catch(cb); + } + return process.nextTick(() => cb()); + }, + cb => + s3 + .send(new PutBucketVersioningCommand(versioningParams)) + .then(() => cb()) + .catch(cb), + ], + cb, + ); } describe('Versioning on a replication source bucket', () => { @@ -69,35 +76,42 @@ describe('Versioning on a replication source bucket', () => { const s3 = bucketUtil.s3; beforeEach(done => { - async.waterfall([ - cb => s3.send(new CreateBucketCommand({ Bucket: bucketName })) - .then(() => cb()) - .catch(cb), - cb => s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: { - Status: 'Enabled', - }, - })) - .then(() => cb()) - .catch(cb), - ], done); + async.waterfall( + [ + cb => + s3 + .send(new CreateBucketCommand({ Bucket: bucketName })) + .then(() => cb()) + .catch(cb), + cb => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: { + Status: 'Enabled', + }, + }), + ) + .then(() => cb()) + .catch(cb), + ], + done, + ); }); afterEach(async () => { await s3.send(new DeleteBucketCommand({ Bucket: bucketName })); }); - it('should not be able to disable versioning if replication enabled', - done => { + it('should not be able to disable versioning if replication enabled', done => { testVersioning(s3, 'Suspended', 'Enabled', false, err => { checkError(err, 'InvalidBucketState'); done(); }); }); - it('should be able to suspend versioning if replication disabled', - done => { + it('should be able to suspend versioning if replication disabled', done => { testVersioning(s3, 'Suspended', 'Disabled', false, err => { checkNoError(err); done(); diff --git a/tests/functional/aws-node-sdk/test/versioning/versioningGeneral1.js b/tests/functional/aws-node-sdk/test/versioning/versioningGeneral1.js index ecba087ebc..3b1be507af 100644 --- a/tests/functional/aws-node-sdk/test/versioning/versioningGeneral1.js +++ b/tests/functional/aws-node-sdk/test/versioning/versioningGeneral1.js @@ -63,111 +63,134 @@ describe('aws-node-sdk test bucket versioning listing', function testSuite() { const keycount = 20; const versioncount = 20; const value = '{"foo":"bar"}'; - async.timesLimit(keycount, 10, (i, next1) => { - const key = `foo${i}`; - masterVersions.push(key); - const params = { Bucket: bucket, Key: key, Body: value }; - async.timesLimit(versioncount, 10, (j, next2) => - s3.send(new PutObjectCommand(params)) - .then(data => { - assert(data.VersionId, 'invalid versionId'); - allVersions.push({ Key: key, VersionId: data.VersionId }); - next2(); - }) - .catch(next2), - next1); - }, err => { - assert.strictEqual(err, null); - assert.strictEqual(allVersions.length, keycount * versioncount); - done(); - }); + async.timesLimit( + keycount, + 10, + (i, next1) => { + const key = `foo${i}`; + masterVersions.push(key); + const params = { Bucket: bucket, Key: key, Body: value }; + async.timesLimit( + versioncount, + 10, + (j, next2) => + s3 + .send(new PutObjectCommand(params)) + .then(data => { + assert(data.VersionId, 'invalid versionId'); + allVersions.push({ Key: key, VersionId: data.VersionId }); + next2(); + }) + .catch(next2), + next1, + ); + }, + err => { + assert.strictEqual(err, null); + assert.strictEqual(allVersions.length, keycount * versioncount); + done(); + }, + ); }); it('should list all latest versions', async () => { const params = { Bucket: bucket, MaxKeys: 1000, Delimiter: '/' }; const data = await s3.send(new ListObjectsCommand(params)); const keys = data.Contents.map(entry => entry.Key); - assert.deepStrictEqual(keys.sort(), masterVersions.sort(), - 'not same keys'); + assert.deepStrictEqual(keys.sort(), masterVersions.sort(), 'not same keys'); }); it('should create some delete markers', done => { const keycount = 15; - async.times(keycount, (i, next) => { - const key = masterVersions[i]; - const params = { Bucket: bucket, Key: key }; - s3.send(new DeleteObjectCommand(params)) - .then(data => { - assert(data.VersionId, 'invalid versionId'); - allVersions.push({ Key: key, VersionId: data.VersionId }); - next(); - }) - .catch(next); - }, done); + async.times( + keycount, + (i, next) => { + const key = masterVersions[i]; + const params = { Bucket: bucket, Key: key }; + s3.send(new DeleteObjectCommand(params)) + .then(data => { + assert(data.VersionId, 'invalid versionId'); + allVersions.push({ Key: key, VersionId: data.VersionId }); + next(); + }) + .catch(next); + }, + done, + ); }); it('should list all latest versions', async () => { const params = { Bucket: bucket, MaxKeys: 1000, Delimiter: '/' }; const data = await s3.send(new ListObjectsCommand(params)); const keys = data.Contents.map(entry => entry.Key); - assert.deepStrictEqual(keys.sort(), masterVersions.sort().slice(15), - 'not same keys'); + assert.deepStrictEqual(keys.sort(), masterVersions.sort().slice(15), 'not same keys'); }); it('should list all versions', done => { const versions = []; const params = { Bucket: bucket, MaxKeys: 15, Delimiter: '/' }; - - async.retry(100, done => { - s3.send(new ListObjectVersionsCommand(params)) - .then(data => { - if (data.Versions) { - data.Versions.forEach(version => versions.push({ - Key: version.Key, VersionId: version.VersionId })); - } - if (data.DeleteMarkers) { - data.DeleteMarkers.forEach(version => versions.push({ - Key: version.Key, VersionId: version.VersionId })); - } - if (data.IsTruncated) { - params.KeyMarker = data.NextKeyMarker; - params.VersionIdMarker = data.NextVersionIdMarker; - return done('not done yet'); - } - return done(); - }) - .catch(err => { - done(err); - }); - }, err => { - if (err) { - return done(err); - } - - assert.deepStrictEqual(versions.sort(comp), allVersions.sort(comp), - 'not same versions'); - - const objectsToDelete = versions - .filter(v => v && v.Key && v.VersionId) - .map(v => ({ - Key: String(v.Key), - VersionId: String(v.VersionId), - })); - - const deleteParams = { - Bucket: bucket, - Delete: { - Objects: objectsToDelete, - } - }; - return s3.send(new DeleteObjectsCommand(deleteParams)) - .then(() => { - done(); - }) - .catch(err => { - done(err); - }); - }); + + async.retry( + 100, + done => { + s3.send(new ListObjectVersionsCommand(params)) + .then(data => { + if (data.Versions) { + data.Versions.forEach(version => + versions.push({ + Key: version.Key, + VersionId: version.VersionId, + }), + ); + } + if (data.DeleteMarkers) { + data.DeleteMarkers.forEach(version => + versions.push({ + Key: version.Key, + VersionId: version.VersionId, + }), + ); + } + if (data.IsTruncated) { + params.KeyMarker = data.NextKeyMarker; + params.VersionIdMarker = data.NextVersionIdMarker; + return done('not done yet'); + } + return done(); + }) + .catch(err => { + done(err); + }); + }, + err => { + if (err) { + return done(err); + } + + assert.deepStrictEqual(versions.sort(comp), allVersions.sort(comp), 'not same versions'); + + const objectsToDelete = versions + .filter(v => v && v.Key && v.VersionId) + .map(v => ({ + Key: String(v.Key), + VersionId: String(v.VersionId), + })); + + const deleteParams = { + Bucket: bucket, + Delete: { + Objects: objectsToDelete, + }, + }; + return s3 + .send(new DeleteObjectsCommand(deleteParams)) + .then(() => { + done(); + }) + .catch(err => { + done(err); + }); + }, + ); }); }); - diff --git a/tests/functional/aws-node-sdk/test/versioning/versioningGeneral2.js b/tests/functional/aws-node-sdk/test/versioning/versioningGeneral2.js index a7dc1b7ee6..d95c701ee9 100644 --- a/tests/functional/aws-node-sdk/test/versioning/versioningGeneral2.js +++ b/tests/functional/aws-node-sdk/test/versioning/versioningGeneral2.js @@ -41,15 +41,14 @@ describe('aws-node-sdk test bucket versioning', function testSuite() { }) .catch(error => { assert.strictEqual(error.$metadata?.httpStatusCode, 400); - assert.strictEqual( - error.name, 'IllegalVersioningConfigurationException'); + assert.strictEqual(error.name, 'IllegalVersioningConfigurationException'); done(); }); }); it('should retrieve an empty versioning configuration', async () => { const params = { Bucket: bucket }; - const {$metadata, ...data} = await s3.send(new GetBucketVersioningCommand(params)); + const { $metadata, ...data } = await s3.send(new GetBucketVersioningCommand(params)); assert.strictEqual($metadata?.httpStatusCode, 200); assert.deepStrictEqual(data, {}); }); @@ -67,15 +66,14 @@ describe('aws-node-sdk test bucket versioning', function testSuite() { }) .catch(error => { assert.strictEqual(error.$metadata?.httpStatusCode, 400); - assert.strictEqual( - error.name, 'IllegalVersioningConfigurationException'); + assert.strictEqual(error.name, 'IllegalVersioningConfigurationException'); done(); }); }); it('should retrieve an empty versioning configuration', async () => { const params = { Bucket: bucket }; - const {$metadata, ...data} = await s3.send(new GetBucketVersioningCommand(params)); + const { $metadata, ...data } = await s3.send(new GetBucketVersioningCommand(params)); assert.strictEqual($metadata?.httpStatusCode, 200); assert.deepStrictEqual(data, {}); }); @@ -85,7 +83,7 @@ describe('aws-node-sdk test bucket versioning', function testSuite() { Bucket: bucket, VersioningConfiguration: { MFADelete: 'fun', - Status: 'let\'s do it', + Status: "let's do it", }, }; s3.send(new PutBucketVersioningCommand(params)) @@ -94,15 +92,14 @@ describe('aws-node-sdk test bucket versioning', function testSuite() { }) .catch(error => { assert.strictEqual(error.$metadata?.httpStatusCode, 400); - assert.strictEqual( - error.name, 'IllegalVersioningConfigurationException'); + assert.strictEqual(error.name, 'IllegalVersioningConfigurationException'); done(); }); }); it('should retrieve an empty versioning configuration', async () => { const params = { Bucket: bucket }; - const {$metadata, ...data} = await s3.send(new GetBucketVersioningCommand(params)); + const { $metadata, ...data } = await s3.send(new GetBucketVersioningCommand(params)); assert.strictEqual($metadata?.httpStatusCode, 200); assert.deepStrictEqual(data, {}); }); @@ -140,8 +137,7 @@ describe('aws-node-sdk test bucket versioning', function testSuite() { return s3.send(new GetObjectCommand(params)); }) .then(data => { - assert.strictEqual(params.VersionId, data.VersionId, - 'version ids are not equal'); + assert.strictEqual(params.VersionId, data.VersionId, 'version ids are not equal'); // TODO compare the value of null version and the original // version when find out how to include value in the put params.VersionId = 'null'; @@ -156,24 +152,28 @@ describe('aws-node-sdk test bucket versioning', function testSuite() { const paramsNull = { Bucket: bucket, Key: '/', VersionId: 'null' }; let nullVersionId; // create new versions - async.timesSeries(counter, (i, next) => { - s3.send(new PutObjectCommand(params)) - .then(data => { - versionIds.push(data.VersionId); - // get the 'null' version - return s3.send(new GetObjectCommand(paramsNull)); - }) - .then(data => { - if (nullVersionId === undefined) { - nullVersionId = data.VersionId; - } - // what to expect: nullVersionId should be the same - assert(nullVersionId, 'nullVersionId should be valid'); - assert.strictEqual(nullVersionId, data.VersionId); - next(); - }) - .catch(next); - }, done); + async.timesSeries( + counter, + (i, next) => { + s3.send(new PutObjectCommand(params)) + .then(data => { + versionIds.push(data.VersionId); + // get the 'null' version + return s3.send(new GetObjectCommand(paramsNull)); + }) + .then(data => { + if (nullVersionId === undefined) { + nullVersionId = data.VersionId; + } + // what to expect: nullVersionId should be the same + assert(nullVersionId, 'nullVersionId should be valid'); + assert.strictEqual(nullVersionId, data.VersionId); + next(); + }) + .catch(next); + }, + done, + ); }); it('should accept valid versioning configuration', async () => { @@ -195,32 +195,41 @@ describe('aws-node-sdk test bucket versioning', function testSuite() { it('should update null version in versioning suspended bucket', done => { const params = { Bucket: bucket, Key: '/' }; const paramsNull = { Bucket: bucket, Key: '/', VersionId: 'null' }; - - async.waterfall([ - callback => s3.send(new GetObjectCommand(paramsNull)) - .then(() => callback()) - .catch(callback), - callback => s3.send(new PutObjectCommand(params)) - .then(() => { - versionIds.push('null'); - callback(); - }) - .catch(callback), - callback => s3.send(new GetObjectCommand(paramsNull)) - .then(data => { - assert.strictEqual(data.VersionId, 'null', - 'version ids are equal'); - callback(); - }) - .catch(callback), - callback => s3.send(new GetObjectCommand(params)) - .then(data => { - assert.strictEqual(data.VersionId, 'null', - 'version ids are not equal'); - callback(); - }) - .catch(callback), - ], done); + + async.waterfall( + [ + callback => + s3 + .send(new GetObjectCommand(paramsNull)) + .then(() => callback()) + .catch(callback), + callback => + s3 + .send(new PutObjectCommand(params)) + .then(() => { + versionIds.push('null'); + callback(); + }) + .catch(callback), + callback => + s3 + .send(new GetObjectCommand(paramsNull)) + .then(data => { + assert.strictEqual(data.VersionId, 'null', 'version ids are equal'); + callback(); + }) + .catch(callback), + callback => + s3 + .send(new GetObjectCommand(params)) + .then(data => { + assert.strictEqual(data.VersionId, 'null', 'version ids are not equal'); + callback(); + }) + .catch(callback), + ], + done, + ); }); it('should enable versioning and preserve the null version', done => { @@ -233,67 +242,84 @@ describe('aws-node-sdk test bucket versioning', function testSuite() { const params = { Bucket: bucket, Key: '/' }; const paramsNull = { Bucket: bucket, Key: '/', VersionId: 'null' }; let nullVersionId; - - async.waterfall([ - callback => s3.send(new GetObjectCommand(paramsNull)) - .then(data => { - nullVersionId = data.VersionId; - callback(); - }) - .catch(callback), - callback => s3.send(new PutBucketVersioningCommand(paramsVersioning)) - .then(() => callback()) - .catch(callback), - callback => async.timesSeries(counter, (i, next) => - s3.send(new PutObjectCommand(params)) - .then(data => { - versionIds.push(data.VersionId); - next(); - }) - .catch(next), - err => callback(err)), - callback => s3.send(new GetObjectCommand(paramsNull)) - .then(data => { - assert.strictEqual(nullVersionId, data.VersionId, - 'version ids are not equal'); - callback(); - }) - .catch(callback), - ], done); + + async.waterfall( + [ + callback => + s3 + .send(new GetObjectCommand(paramsNull)) + .then(data => { + nullVersionId = data.VersionId; + callback(); + }) + .catch(callback), + callback => + s3 + .send(new PutBucketVersioningCommand(paramsVersioning)) + .then(() => callback()) + .catch(callback), + callback => + async.timesSeries( + counter, + (i, next) => + s3 + .send(new PutObjectCommand(params)) + .then(data => { + versionIds.push(data.VersionId); + next(); + }) + .catch(next), + err => callback(err), + ), + callback => + s3 + .send(new GetObjectCommand(paramsNull)) + .then(data => { + assert.strictEqual(nullVersionId, data.VersionId, 'version ids are not equal'); + callback(); + }) + .catch(callback), + ], + done, + ); }); it('should create delete marker and keep the null version', done => { const params = { Bucket: bucket, Key: '/' }; const paramsNull = { Bucket: bucket, Key: '/', VersionId: 'null' }; - + s3.send(new GetObjectCommand(paramsNull)) .then(data => { const nullVersionId = data.VersionId; - async.timesSeries(counter, (i, next) => { - s3.send(new DeleteObjectCommand(params)) - .then(data => { - versionIds.push(data.VersionId); - return s3.send(new GetObjectCommand(params)); - }) - .then(() => { - next(new Error('Expected NoSuchKey error')); - }) - .catch(err => { - assert.strictEqual(err.name, 'NoSuchKey'); - next(); - }); - }, err => { - if (err) { - return done(err); - } - return s3.send(new GetObjectCommand(paramsNull)) - .then(data => { - assert.strictEqual(nullVersionId, data.VersionId, - 'version ids are not equal'); - done(); - }) - .catch(done); - }); + async.timesSeries( + counter, + (i, next) => { + s3.send(new DeleteObjectCommand(params)) + .then(data => { + versionIds.push(data.VersionId); + return s3.send(new GetObjectCommand(params)); + }) + .then(() => { + next(new Error('Expected NoSuchKey error')); + }) + .catch(err => { + assert.strictEqual(err.name, 'NoSuchKey'); + next(); + }); + }, + err => { + if (err) { + return done(err); + } + return s3 + .send(new GetObjectCommand(paramsNull)) + .then(data => { + assert.strictEqual(nullVersionId, data.VersionId, 'version ids are not equal'); + done(); + }) + .catch(done); + }, + ); }) .catch(done); }); @@ -301,29 +327,30 @@ describe('aws-node-sdk test bucket versioning', function testSuite() { it('should delete latest version and get the next version', done => { versionIds.reverse(); const params = { Bucket: bucket, Key: '/' }; - - async.timesSeries(versionIds.length, (i, next) => { - const versionId = versionIds[i]; - const nextVersionId = i < versionIds.length - 1 ? - versionIds[i + 1] : undefined; - const paramsVersion = - { Bucket: bucket, Key: '/', VersionId: versionId }; - - s3.send(new DeleteObjectCommand(paramsVersion)) - .then(() => s3.send(new GetObjectCommand(params))) - .then(data => { - assert(data.VersionId, 'invalid versionId'); - if (nextVersionId !== 'null') { - assert.strictEqual(data.VersionId, nextVersionId); - } - next(); - }) - .catch(err => { - assert(err.name === 'NotFound' || - err.name === 'NoSuchKey', 'error'); - next(); - }); - }, done); + + async.timesSeries( + versionIds.length, + (i, next) => { + const versionId = versionIds[i]; + const nextVersionId = i < versionIds.length - 1 ? versionIds[i + 1] : undefined; + const paramsVersion = { Bucket: bucket, Key: '/', VersionId: versionId }; + + s3.send(new DeleteObjectCommand(paramsVersion)) + .then(() => s3.send(new GetObjectCommand(params))) + .then(data => { + assert(data.VersionId, 'invalid versionId'); + if (nextVersionId !== 'null') { + assert.strictEqual(data.VersionId, nextVersionId); + } + next(); + }) + .catch(err => { + assert(err.name === 'NotFound' || err.name === 'NoSuchKey', 'error'); + next(); + }); + }, + done, + ); }); it('should create a bunch of objects and their versions', done => { @@ -331,38 +358,49 @@ describe('aws-node-sdk test bucket versioning', function testSuite() { const keycount = 50; const versioncount = 20; const value = '{"foo":"bar"}'; - - async.timesLimit(keycount, 10, (i, next1) => { - const key = `foo${i}`; - const params = { Bucket: bucket, Key: key, Body: value }; - async.timesLimit(versioncount, 10, (j, next2) => - s3.send(new PutObjectCommand(params)) - .then(data => { - assert(data.VersionId, 'invalid versionId'); - vids.push({ Key: key, VersionId: data.VersionId }); - next2(); - }) - .catch(next2), - next1); - }, err => { - if (err) { - return done(err); - } - assert.strictEqual(vids.length, keycount * versioncount); - const params = { - Bucket: bucket, - Delete: { - Objects: vids.map(v => ({ - Key: v.Key, - VersionId: v.VersionId, - })), - } - }; - // TODO use delete marker and check with the result - process.stdout.write('creating objects done, now deleting...'); - return s3.send(new DeleteObjectsCommand(params)) - .then(() => done()) - .catch(done); - }); + + async.timesLimit( + keycount, + 10, + (i, next1) => { + const key = `foo${i}`; + const params = { Bucket: bucket, Key: key, Body: value }; + async.timesLimit( + versioncount, + 10, + (j, next2) => + s3 + .send(new PutObjectCommand(params)) + .then(data => { + assert(data.VersionId, 'invalid versionId'); + vids.push({ Key: key, VersionId: data.VersionId }); + next2(); + }) + .catch(next2), + next1, + ); + }, + err => { + if (err) { + return done(err); + } + assert.strictEqual(vids.length, keycount * versioncount); + const params = { + Bucket: bucket, + Delete: { + Objects: vids.map(v => ({ + Key: v.Key, + VersionId: v.VersionId, + })), + }, + }; + // TODO use delete marker and check with the result + process.stdout.write('creating objects done, now deleting...'); + return s3 + .send(new DeleteObjectsCommand(params)) + .then(() => done()) + .catch(done); + }, + ); }); }); diff --git a/tests/functional/backbeat/bucketIndexing.js b/tests/functional/backbeat/bucketIndexing.js index b0bd57d7af..33e0c6409b 100644 --- a/tests/functional/backbeat/bucketIndexing.js +++ b/tests/functional/backbeat/bucketIndexing.js @@ -1,13 +1,9 @@ const assert = require('assert'); const async = require('async'); -const { - CreateBucketCommand, - DeleteBucketCommand, -} = require('@aws-sdk/client-s3'); +const { CreateBucketCommand, DeleteBucketCommand } = require('@aws-sdk/client-s3'); const { makeRequest } = require('../../functional/raw-node/utils/makeRequest'); -const BucketUtility = - require('../../functional/aws-node-sdk/lib/utility/bucket-util'); +const BucketUtility = require('../../functional/aws-node-sdk/lib/utility/bucket-util'); const ipAddress = process.env.IP ? process.env.IP : '127.0.0.1'; @@ -18,57 +14,63 @@ let credentials = null; let backbeatAuthCredentials = null; async function getCredentials() { - const creds = await s3.config.credentials(); - credentials = { - accessKey: creds.accessKeyId, - secretKey: creds.secretAccessKey, - }; + const creds = await s3.config.credentials(); + credentials = { + accessKey: creds.accessKeyId, + secretKey: creds.secretAccessKey, + }; return credentials; } const TEST_BUCKET = 'bucket-for-bucket-indexing'; function indexDeleteRequest(payload, bucket, cb) { - makeRequest({ - authCredentials: backbeatAuthCredentials, - hostname: ipAddress, - port: 8000, - method: 'POST', - path: - `/_/backbeat/index/${bucket}`, - headers: {}, - jsonResponse: true, - requestBody: JSON.stringify(payload), - queryObj: { operation: 'delete' }, - }, cb); + makeRequest( + { + authCredentials: backbeatAuthCredentials, + hostname: ipAddress, + port: 8000, + method: 'POST', + path: `/_/backbeat/index/${bucket}`, + headers: {}, + jsonResponse: true, + requestBody: JSON.stringify(payload), + queryObj: { operation: 'delete' }, + }, + cb, + ); } function indexPutRequest(payload, bucket, cb) { - makeRequest({ - authCredentials: backbeatAuthCredentials, - hostname: ipAddress, - port: 8000, - method: 'POST', - path: - `/_/backbeat/index/${bucket}`, - headers: {}, - jsonResponse: true, - requestBody: JSON.stringify(payload), - queryObj: { operation: 'add' }, - }, cb); + makeRequest( + { + authCredentials: backbeatAuthCredentials, + hostname: ipAddress, + port: 8000, + method: 'POST', + path: `/_/backbeat/index/${bucket}`, + headers: {}, + jsonResponse: true, + requestBody: JSON.stringify(payload), + queryObj: { operation: 'add' }, + }, + cb, + ); } function indexGetRequest(bucket, cb) { - makeRequest({ - authCredentials: backbeatAuthCredentials, - hostname: ipAddress, - port: 8000, - method: 'GET', - path: - `/_/backbeat/index/${bucket}`, - headers: {}, - jsonResponse: true, - }, cb); + makeRequest( + { + authCredentials: backbeatAuthCredentials, + hostname: ipAddress, + port: 8000, + method: 'GET', + path: `/_/backbeat/index/${bucket}`, + headers: {}, + jsonResponse: true, + }, + cb, + ); } const indexReqObject = [ @@ -92,9 +94,7 @@ const indexReqObject = [ const indexRespObject = [ { name: '_id_', - keys: [ - { key: '_id', order: 1 }, - ] + keys: [{ key: '_id', order: 1 }], }, { keys: [ @@ -136,19 +136,21 @@ describe('Indexing Routes', () => { }); it('should reject non-authenticated requests', done => { - makeRequest({ - hostname: ipAddress, - port: 8000, - method: 'GET', - path: - '/_/backbeat/index/testbucket', - headers: {}, - jsonResponse: true, - }, err => { - assert(err); - assert.strictEqual(err.code, 'AccessDenied'); - done(); - }); + makeRequest( + { + hostname: ipAddress, + port: 8000, + method: 'GET', + path: '/_/backbeat/index/testbucket', + headers: {}, + jsonResponse: true, + }, + err => { + assert(err); + assert.strictEqual(err.code, 'AccessDenied'); + done(); + }, + ); }); it('should return error: invalid payload - empty', done => { @@ -177,60 +179,66 @@ describe('Indexing Routes', () => { describeIfMongo('with mongodb metadata', () => { it('should successfully add indexes', done => { - async.series([ - next => { - indexPutRequest(indexReqObject, TEST_BUCKET, err => { - assert.ifError(err); - next(); - }); - }, - next => { - indexGetRequest(TEST_BUCKET, (err, data) => { - assert.ifError(err); - const res = JSON.parse(data.body); - assert.deepStrictEqual(res.Indexes, indexRespObject); - next(); - }); - }, - ], done); + async.series( + [ + next => { + indexPutRequest(indexReqObject, TEST_BUCKET, err => { + assert.ifError(err); + next(); + }); + }, + next => { + indexGetRequest(TEST_BUCKET, (err, data) => { + assert.ifError(err); + const res = JSON.parse(data.body); + assert.deepStrictEqual(res.Indexes, indexRespObject); + next(); + }); + }, + ], + done, + ); }); it('should successfully delete indexes', done => { - async.series([ - next => { - indexPutRequest(indexReqObject, TEST_BUCKET, err => { - assert.ifError(err); - next(); - }); - }, - next => { - indexGetRequest(TEST_BUCKET, (err, data) => { - assert.ifError(err); - const res = JSON.parse(data.body); - assert.deepStrictEqual(res.Indexes, indexRespObject); - next(); - }); - }, - next => { - indexDeleteRequest(indexReqObject, TEST_BUCKET, err => { - assert.ifError(err); - next(); - }); - }, - next => { - indexGetRequest(TEST_BUCKET, (err, data) => { - assert.ifError(err); - const res = JSON.parse(data.body); - assert.deepStrictEqual(res.Indexes, [ - { - name: '_id_', - keys: [{ key: '_id', order: 1 }], - } - ]); - next(); - }); - }, - ], done); + async.series( + [ + next => { + indexPutRequest(indexReqObject, TEST_BUCKET, err => { + assert.ifError(err); + next(); + }); + }, + next => { + indexGetRequest(TEST_BUCKET, (err, data) => { + assert.ifError(err); + const res = JSON.parse(data.body); + assert.deepStrictEqual(res.Indexes, indexRespObject); + next(); + }); + }, + next => { + indexDeleteRequest(indexReqObject, TEST_BUCKET, err => { + assert.ifError(err); + next(); + }); + }, + next => { + indexGetRequest(TEST_BUCKET, (err, data) => { + assert.ifError(err); + const res = JSON.parse(data.body); + assert.deepStrictEqual(res.Indexes, [ + { + name: '_id_', + keys: [{ key: '_id', order: 1 }], + }, + ]); + next(); + }); + }, + ], + done, + ); }); }); @@ -263,4 +271,3 @@ describe('Indexing Routes', () => { }); }); }); - diff --git a/tests/functional/backbeat/excludedDataStoreName.js b/tests/functional/backbeat/excludedDataStoreName.js index a4b5498e8f..000f6728d5 100644 --- a/tests/functional/backbeat/excludedDataStoreName.js +++ b/tests/functional/backbeat/excludedDataStoreName.js @@ -18,12 +18,12 @@ const bucketUtil = new BucketUtility('default', {}); const s3 = bucketUtil.s3; async function getCredentials() { - const creds = await s3.config.credentials(); - const credentials = { - accessKey: creds.accessKeyId, - secretKey: creds.secretAccessKey, - }; - return credentials; + const creds = await s3.config.credentials(); + const credentials = { + accessKey: creds.accessKeyId, + secretKey: creds.secretAccessKey, + }; + return credentials; } async function getS3Hostname() { @@ -38,87 +38,96 @@ describe('excludedDataStoreName', () => { let location1; let location2; - before(done => async.series([ - next => { - getCredentials() - .then(creds => { - credentials = creds; - next(); - }) - .catch(next); - }, - next => { - getS3Hostname() - .then(hostname => { - s3Hostname = hostname; - location1 = config.restEndpoints[s3Hostname] || config.restEndpoints.localhost; - location2 = 'us-east-2'; - next(); - }) - .catch(next); - }, - next => { - s3.send(new CreateBucketCommand({ Bucket: testBucket })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new PutBucketVersioningCommand({ - Bucket: testBucket, - VersioningConfiguration: { Status: 'Enabled' }, - })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new PutObjectCommand({ Bucket: testBucket, Key: 'key0' })) - .then(data => { - expectedVersions.push(data.VersionId); - next(); - }) - .catch(next); - }, - next => { - s3.send(new PutObjectCommand({ Bucket: testBucket, Key: 'key0' })) - .then(data => { - const versionId = data.VersionId; - return updateMetadata( - { bucket: testBucket, objectKey: 'key0', versionId, authCredentials: credentials }, - { dataStoreName: location2 }, - next); - }) - .catch(next); - }, - next => { - s3.send(new PutObjectCommand({ Bucket: testBucket, Key: 'key0' })) - .then(data => { - expectedVersions.push(data.VersionId); - next(); - }) - .catch(next); - }, - next => { - s3.send(new PutObjectCommand({ Bucket: testBucket, Key: 'key0' })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new PutObjectCommand({ Bucket: testBucket, Key: 'key1' })) - .then(data => { - const versionId = data.VersionId; - return updateMetadata( - { bucket: testBucket, objectKey: 'key1', versionId, authCredentials: credentials }, - { dataStoreName: location2 }, - next); - }) - .catch(next); - }, - next => { - s3.send(new PutObjectCommand({ Bucket: testBucket, Key: 'key2' })) - .then(() => next()) - .catch(next); - }, - ], done)); + before(done => + async.series( + [ + next => { + getCredentials() + .then(creds => { + credentials = creds; + next(); + }) + .catch(next); + }, + next => { + getS3Hostname() + .then(hostname => { + s3Hostname = hostname; + location1 = config.restEndpoints[s3Hostname] || config.restEndpoints.localhost; + location2 = 'us-east-2'; + next(); + }) + .catch(next); + }, + next => { + s3.send(new CreateBucketCommand({ Bucket: testBucket })) + .then(() => next()) + .catch(next); + }, + next => { + s3.send( + new PutBucketVersioningCommand({ + Bucket: testBucket, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + s3.send(new PutObjectCommand({ Bucket: testBucket, Key: 'key0' })) + .then(data => { + expectedVersions.push(data.VersionId); + next(); + }) + .catch(next); + }, + next => { + s3.send(new PutObjectCommand({ Bucket: testBucket, Key: 'key0' })) + .then(data => { + const versionId = data.VersionId; + return updateMetadata( + { bucket: testBucket, objectKey: 'key0', versionId, authCredentials: credentials }, + { dataStoreName: location2 }, + next, + ); + }) + .catch(next); + }, + next => { + s3.send(new PutObjectCommand({ Bucket: testBucket, Key: 'key0' })) + .then(data => { + expectedVersions.push(data.VersionId); + next(); + }) + .catch(next); + }, + next => { + s3.send(new PutObjectCommand({ Bucket: testBucket, Key: 'key0' })) + .then(() => next()) + .catch(next); + }, + next => { + s3.send(new PutObjectCommand({ Bucket: testBucket, Key: 'key1' })) + .then(data => { + const versionId = data.VersionId; + return updateMetadata( + { bucket: testBucket, objectKey: 'key1', versionId, authCredentials: credentials }, + { dataStoreName: location2 }, + next, + ); + }) + .catch(next); + }, + next => { + s3.send(new PutObjectCommand({ Bucket: testBucket, Key: 'key2' })) + .then(() => next()) + .catch(next); + }, + ], + done, + ), + ); after(async () => { await removeAllVersionsPromise({ Bucket: testBucket }); @@ -126,183 +135,182 @@ describe('excludedDataStoreName', () => { }); it('should return error when listing current versions if excluded-data-store-name is not in config', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'current', 'excluded-data-store-name': 'idonotexist' }, - authCredentials: credentials, - }, err => { - assert(err, 'Expected error but found none'); - assert.strictEqual(err.code, 'InvalidLocationConstraint'); - assert.strictEqual(err.statusCode, 400); - assert.strictEqual(err.message, 'value of the location you are attempting to set' + - ' - idonotexist - is not listed in the locationConstraint config'); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'current', 'excluded-data-store-name': 'idonotexist' }, + authCredentials: credentials, + }, + err => { + assert(err, 'Expected error but found none'); + assert.strictEqual(err.code, 'InvalidLocationConstraint'); + assert.strictEqual(err.statusCode, 400); + assert.strictEqual( + err.message, + 'value of the location you are attempting to set' + + ' - idonotexist - is not listed in the locationConstraint config', + ); + return done(); + }, + ); }); it('should return error when listing non-current versions if excluded-data-store-name is not in config', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'noncurrent', 'excluded-data-store-name': 'idonotexist' }, - authCredentials: credentials, - }, err => { - assert(err, 'Expected error but found none'); - assert.strictEqual(err.code, 'InvalidLocationConstraint'); - assert.strictEqual(err.statusCode, 400); - assert.strictEqual(err.message, 'value of the location you are attempting to set' + - ' - idonotexist - is not listed in the locationConstraint config'); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'noncurrent', 'excluded-data-store-name': 'idonotexist' }, + authCredentials: credentials, + }, + err => { + assert(err, 'Expected error but found none'); + assert.strictEqual(err.code, 'InvalidLocationConstraint'); + assert.strictEqual(err.statusCode, 400); + assert.strictEqual( + err.message, + 'value of the location you are attempting to set' + + ' - idonotexist - is not listed in the locationConstraint config', + ); + return done(); + }, + ); }); it('should exclude current versions stored in location2', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'current', 'excluded-data-store-name': location2 }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); - - assert.strictEqual(data.IsTruncated, false); - assert(!data.NextMarker); - assert.strictEqual(data.MaxKeys, 1000); - - const contents = data.Contents; - assert.strictEqual(contents.length, 2); - - assert.strictEqual(contents[0].Key, 'key0'); - assert.strictEqual(contents[0].DataStoreName, location1); - assert.strictEqual(contents[1].Key, 'key2'); - assert.strictEqual(contents[1].DataStoreName, location1); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'current', 'excluded-data-store-name': location2 }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); + + assert.strictEqual(data.IsTruncated, false); + assert(!data.NextMarker); + assert.strictEqual(data.MaxKeys, 1000); + + const contents = data.Contents; + assert.strictEqual(contents.length, 2); + + assert.strictEqual(contents[0].Key, 'key0'); + assert.strictEqual(contents[0].DataStoreName, location1); + assert.strictEqual(contents[1].Key, 'key2'); + assert.strictEqual(contents[1].DataStoreName, location1); + return done(); + }, + ); }); it('should return trucated listing that excludes current versions stored in location2', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'current', 'excluded-data-store-name': location2, 'max-keys': '1' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); - - assert.strictEqual(data.IsTruncated, true); - assert.strictEqual(data.NextMarker, 'key0'); - assert.strictEqual(data.MaxKeys, 1); - - const contents = data.Contents; - assert.strictEqual(contents.length, 1); - - assert.strictEqual(contents[0].Key, 'key0'); - assert.strictEqual(contents[0].DataStoreName, location1); - - return makeBackbeatRequest({ + makeBackbeatRequest( + { method: 'GET', bucket: testBucket, - queryObj: { - 'list-type': 'current', - 'excluded-data-store-name': location2, - 'max-keys': '1', - 'marker': 'key0', - }, + queryObj: { 'list-type': 'current', 'excluded-data-store-name': location2, 'max-keys': '1' }, authCredentials: credentials, - }, (err, response) => { + }, + (err, response) => { assert.ifError(err); assert.strictEqual(response.statusCode, 200); const data = JSON.parse(response.body); - assert.strictEqual(data.IsTruncated, false); - assert(!data.NextMarker); + assert.strictEqual(data.IsTruncated, true); + assert.strictEqual(data.NextMarker, 'key0'); assert.strictEqual(data.MaxKeys, 1); const contents = data.Contents; assert.strictEqual(contents.length, 1); - assert.strictEqual(contents[0].Key, 'key2'); + assert.strictEqual(contents[0].Key, 'key0'); assert.strictEqual(contents[0].DataStoreName, location1); - return done(); - }); - }); + + return makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { + 'list-type': 'current', + 'excluded-data-store-name': location2, + 'max-keys': '1', + marker: 'key0', + }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); + + assert.strictEqual(data.IsTruncated, false); + assert(!data.NextMarker); + assert.strictEqual(data.MaxKeys, 1); + + const contents = data.Contents; + assert.strictEqual(contents.length, 1); + + assert.strictEqual(contents[0].Key, 'key2'); + assert.strictEqual(contents[0].DataStoreName, location1); + return done(); + }, + ); + }, + ); }); it('should exclude non-current versions stored in location2', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'noncurrent', 'excluded-data-store-name': location2 }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); - - assert.strictEqual(data.IsTruncated, false); - assert(!data.NextKeyMarker); - assert(!data.NextVersionIdMarker); - assert.strictEqual(data.MaxKeys, 1000); - - const contents = data.Contents; - assert.strictEqual(contents.length, 2); - - assert.strictEqual(contents[0].Key, 'key0'); - assert.strictEqual(contents[0].DataStoreName, location1); - assert.strictEqual(contents[0].VersionId, expectedVersions[1]); - assert.strictEqual(contents[1].Key, 'key0'); - assert.strictEqual(contents[1].DataStoreName, location1); - assert.strictEqual(contents[1].VersionId, expectedVersions[0]); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'noncurrent', 'excluded-data-store-name': location2 }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); + + assert.strictEqual(data.IsTruncated, false); + assert(!data.NextKeyMarker); + assert(!data.NextVersionIdMarker); + assert.strictEqual(data.MaxKeys, 1000); + + const contents = data.Contents; + assert.strictEqual(contents.length, 2); + + assert.strictEqual(contents[0].Key, 'key0'); + assert.strictEqual(contents[0].DataStoreName, location1); + assert.strictEqual(contents[0].VersionId, expectedVersions[1]); + assert.strictEqual(contents[1].Key, 'key0'); + assert.strictEqual(contents[1].DataStoreName, location1); + assert.strictEqual(contents[1].VersionId, expectedVersions[0]); + return done(); + }, + ); }); it('should return trucated listing that excludes non-current versions stored in location2', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'noncurrent', 'excluded-data-store-name': location2, 'max-keys': '1' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); - - assert.strictEqual(data.IsTruncated, true); - assert.strictEqual(data.NextKeyMarker, 'key0'); - assert.strictEqual(data.NextVersionIdMarker, expectedVersions[1]); - assert.strictEqual(data.MaxKeys, 1); - - const contents = data.Contents; - assert.strictEqual(contents.length, 1); - - assert.strictEqual(contents[0].Key, 'key0'); - assert.strictEqual(contents[0].DataStoreName, location1); - assert.strictEqual(contents[0].VersionId, expectedVersions[1]); - return makeBackbeatRequest({ + makeBackbeatRequest( + { method: 'GET', bucket: testBucket, - queryObj: { - 'list-type': 'noncurrent', - 'excluded-data-store-name': location2, - 'key-marker': 'key0', - 'version-id-marker': expectedVersions[1], - 'max-keys': '1', - }, + queryObj: { 'list-type': 'noncurrent', 'excluded-data-store-name': location2, 'max-keys': '1' }, authCredentials: credentials, - }, (err, response) => { + }, + (err, response) => { assert.ifError(err); assert.strictEqual(response.statusCode, 200); const data = JSON.parse(response.body); assert.strictEqual(data.IsTruncated, true); assert.strictEqual(data.NextKeyMarker, 'key0'); - assert.strictEqual(data.NextVersionIdMarker, expectedVersions[0]); + assert.strictEqual(data.NextVersionIdMarker, expectedVersions[1]); assert.strictEqual(data.MaxKeys, 1); const contents = data.Contents; @@ -310,33 +318,67 @@ describe('excludedDataStoreName', () => { assert.strictEqual(contents[0].Key, 'key0'); assert.strictEqual(contents[0].DataStoreName, location1); - assert.strictEqual(contents[0].VersionId, expectedVersions[0]); - return makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { - 'list-type': 'noncurrent', - 'excluded-data-store-name': location2, - 'key-marker': 'key0', - 'version-id-marker': expectedVersions[0], - 'max-keys': '1', + assert.strictEqual(contents[0].VersionId, expectedVersions[1]); + return makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { + 'list-type': 'noncurrent', + 'excluded-data-store-name': location2, + 'key-marker': 'key0', + 'version-id-marker': expectedVersions[1], + 'max-keys': '1', + }, + authCredentials: credentials, }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); - - assert.strictEqual(data.IsTruncated, false); - assert(!data.NextKeyMarker); - assert(!data.NextVersionIdMarker); - assert.strictEqual(data.MaxKeys, 1); - - const contents = data.Contents; - assert.strictEqual(contents.length, 0); - return done(); - }); - }); - }); + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); + + assert.strictEqual(data.IsTruncated, true); + assert.strictEqual(data.NextKeyMarker, 'key0'); + assert.strictEqual(data.NextVersionIdMarker, expectedVersions[0]); + assert.strictEqual(data.MaxKeys, 1); + + const contents = data.Contents; + assert.strictEqual(contents.length, 1); + + assert.strictEqual(contents[0].Key, 'key0'); + assert.strictEqual(contents[0].DataStoreName, location1); + assert.strictEqual(contents[0].VersionId, expectedVersions[0]); + return makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { + 'list-type': 'noncurrent', + 'excluded-data-store-name': location2, + 'key-marker': 'key0', + 'version-id-marker': expectedVersions[0], + 'max-keys': '1', + }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); + + assert.strictEqual(data.IsTruncated, false); + assert(!data.NextKeyMarker); + assert(!data.NextVersionIdMarker); + assert.strictEqual(data.MaxKeys, 1); + + const contents = data.Contents; + assert.strictEqual(contents.length, 0); + return done(); + }, + ); + }, + ); + }, + ); }); }); diff --git a/tests/functional/backbeat/listDeleteMarker.js b/tests/functional/backbeat/listDeleteMarker.js index d0ee845bdd..b54072b641 100644 --- a/tests/functional/backbeat/listDeleteMarker.js +++ b/tests/functional/backbeat/listDeleteMarker.js @@ -26,41 +26,57 @@ async function getCredentials() { return credentials; } - describe('listLifecycle with non-current delete marker', () => { let expectedVersionId; let expectedDMVersionId; const testBucket = 'bucket-for-list-lifecycle-noncurrent-dm-tests'; const keyName = 'key0'; - before(done => async.series([ - next => getCredentials().then(creds => { - credentials = creds; - next(); - }).catch(next), - next => s3.send(new CreateBucketCommand({ Bucket: testBucket })) - .then(() => next()) - .catch(next), - next => s3.send(new PutBucketVersioningCommand({ - Bucket: testBucket, - VersioningConfiguration: { Status: 'Enabled' }, - })) - .then(() => next()) - .catch(next), - next => s3.send(new DeleteObjectCommand({ Bucket: testBucket, Key: keyName })) - .then(data => { - expectedDMVersionId = data.VersionId; - next(); - }) - .catch(next), - next => s3.send(new PutObjectCommand({ Bucket: testBucket, Key: keyName })) - .then(data => { - expectedVersionId = data.VersionId; - next(); - }) - .catch(next), - ], done)); - + before(done => + async.series( + [ + next => + getCredentials() + .then(creds => { + credentials = creds; + next(); + }) + .catch(next), + next => + s3 + .send(new CreateBucketCommand({ Bucket: testBucket })) + .then(() => next()) + .catch(next), + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: testBucket, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ) + .then(() => next()) + .catch(next), + next => + s3 + .send(new DeleteObjectCommand({ Bucket: testBucket, Key: keyName })) + .then(data => { + expectedDMVersionId = data.VersionId; + next(); + }) + .catch(next), + next => + s3 + .send(new PutObjectCommand({ Bucket: testBucket, Key: keyName })) + .then(data => { + expectedVersionId = data.VersionId; + next(); + }) + .catch(next), + ], + done, + ), + ); after(async () => { await removeAllVersionsPromise({ Bucket: testBucket }); @@ -68,66 +84,75 @@ describe('listLifecycle with non-current delete marker', () => { }); it('should return the current version', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'current' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); - - assert.strictEqual(data.IsTruncated, false); - assert(!data.NextKeyMarker); - assert.strictEqual(data.MaxKeys, 1000); - assert.strictEqual(data.Contents.length, 1); - const key = data.Contents[0]; - assert.strictEqual(key.Key, keyName); - assert.strictEqual(key.VersionId, expectedVersionId); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'current' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); + + assert.strictEqual(data.IsTruncated, false); + assert(!data.NextKeyMarker); + assert.strictEqual(data.MaxKeys, 1000); + assert.strictEqual(data.Contents.length, 1); + const key = data.Contents[0]; + assert.strictEqual(key.Key, keyName); + assert.strictEqual(key.VersionId, expectedVersionId); + return done(); + }, + ); }); it('should return the non-current delete marker', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'noncurrent' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); - - assert.strictEqual(data.IsTruncated, false); - assert(!data.NextKeyMarker); - assert.strictEqual(data.MaxKeys, 1000); - assert.strictEqual(data.Contents.length, 1); - const key = data.Contents[0]; - assert.strictEqual(key.Key, keyName); - assert.strictEqual(key.VersionId, expectedDMVersionId); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'noncurrent' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); + + assert.strictEqual(data.IsTruncated, false); + assert(!data.NextKeyMarker); + assert.strictEqual(data.MaxKeys, 1000); + assert.strictEqual(data.Contents.length, 1); + const key = data.Contents[0]; + assert.strictEqual(key.Key, keyName); + assert.strictEqual(key.VersionId, expectedDMVersionId); + return done(); + }, + ); }); it('should return no orphan delete marker', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'orphan' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); - - assert.strictEqual(data.IsTruncated, false); - assert(!data.NextKeyMarker); - assert.strictEqual(data.MaxKeys, 1000); - assert.strictEqual(data.Contents.length, 0); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'orphan' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); + + assert.strictEqual(data.IsTruncated, false); + assert(!data.NextKeyMarker); + assert.strictEqual(data.MaxKeys, 1000); + assert.strictEqual(data.Contents.length, 0); + return done(); + }, + ); }); }); @@ -136,26 +161,41 @@ describe('listLifecycle with current delete marker version', () => { const testBucket = 'bucket-for-list-lifecycle-current-dm-tests'; const keyName = 'key0'; - before(done => async.series([ - next => s3.send(new CreateBucketCommand({ Bucket: testBucket })) - .then(() => next()) - .catch(next), - next => s3.send(new PutBucketVersioningCommand({ - Bucket: testBucket, - VersioningConfiguration: { Status: 'Enabled' }, - })) - .then(() => next()) - .catch(next), - next => s3.send(new PutObjectCommand({ Bucket: testBucket, Key: keyName })) - .then(data => { - expectedVersionId = data.VersionId; - next(); - }) - .catch(next), - next => s3.send(new DeleteObjectCommand({ Bucket: testBucket, Key: keyName })) - .then(() => next()) - .catch(next), - ], done)); + before(done => + async.series( + [ + next => + s3 + .send(new CreateBucketCommand({ Bucket: testBucket })) + .then(() => next()) + .catch(next), + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: testBucket, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ) + .then(() => next()) + .catch(next), + next => + s3 + .send(new PutObjectCommand({ Bucket: testBucket, Key: keyName })) + .then(data => { + expectedVersionId = data.VersionId; + next(); + }) + .catch(next), + next => + s3 + .send(new DeleteObjectCommand({ Bucket: testBucket, Key: keyName })) + .then(() => next()) + .catch(next), + ], + done, + ), + ); after(async () => { await removeAllVersionsPromise({ Bucket: testBucket }); @@ -163,62 +203,71 @@ describe('listLifecycle with current delete marker version', () => { }); it('should return no current object if current version is a delete marker', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'current' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); - - assert.strictEqual(data.IsTruncated, false); - assert(!data.NextKeyMarker); - assert.strictEqual(data.MaxKeys, 1000); - assert.strictEqual(data.Contents.length, 0); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'current' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); + + assert.strictEqual(data.IsTruncated, false); + assert(!data.NextKeyMarker); + assert.strictEqual(data.MaxKeys, 1000); + assert.strictEqual(data.Contents.length, 0); + return done(); + }, + ); }); it('should return the non-current version', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'noncurrent' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); - - assert.strictEqual(data.IsTruncated, false); - assert(!data.NextKeyMarker); - assert.strictEqual(data.MaxKeys, 1000); - assert.strictEqual(data.Contents.length, 1); - const key = data.Contents[0]; - assert.strictEqual(key.Key, keyName); - assert.strictEqual(key.VersionId, expectedVersionId); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'noncurrent' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); + + assert.strictEqual(data.IsTruncated, false); + assert(!data.NextKeyMarker); + assert.strictEqual(data.MaxKeys, 1000); + assert.strictEqual(data.Contents.length, 1); + const key = data.Contents[0]; + assert.strictEqual(key.Key, keyName); + assert.strictEqual(key.VersionId, expectedVersionId); + return done(); + }, + ); }); it('should return no orphan delete marker', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'orphan' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); - - assert.strictEqual(data.IsTruncated, false); - assert(!data.NextKeyMarker); - assert.strictEqual(data.MaxKeys, 1000); - assert.strictEqual(data.Contents.length, 0); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'orphan' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); + + assert.strictEqual(data.IsTruncated, false); + assert(!data.NextKeyMarker); + assert.strictEqual(data.MaxKeys, 1000); + assert.strictEqual(data.Contents.length, 0); + return done(); + }, + ); }); }); diff --git a/tests/functional/backbeat/listLifecycleCurrents.js b/tests/functional/backbeat/listLifecycleCurrents.js index 5b83270a82..7e4821bf51 100644 --- a/tests/functional/backbeat/listLifecycleCurrents.js +++ b/tests/functional/backbeat/listLifecycleCurrents.js @@ -52,10 +52,12 @@ function checkContents(contents, expectedKeyVersions) { assert(d.Owner.ID); assert(d.StorageClass); assert.strictEqual(d.StorageClass, 'STANDARD'); - assert.deepStrictEqual(d.TagSet, [{ - Key: 'mykey', - Value: 'myvalue', - }]); + assert.deepStrictEqual(d.TagSet, [ + { + Key: 'mykey', + Value: 'myvalue', + }, + ]); assert.strictEqual(d.IsLatest, true); assert.strictEqual(d.DataStoreName, location); assert.strictEqual(d.ListType, 'current'); @@ -71,540 +73,685 @@ function checkContents(contents, expectedKeyVersions) { let date; const expectedKeyVersions = {}; - before(done => async.series([ - next => { - getCredentials() - .then(creds => { - credentials = creds; - return getS3Hostname(); - }) - .then(() => next()) - .catch(next); - }, - next => s3.send(new CreateBucketCommand({ Bucket: testBucket })) - .then(() => next()) - .catch(next), - next => s3.send(new CreateBucketCommand({ Bucket: emptyBucket })) - .then(() => next()) - .catch(next), - next => { - if (versioning !== 'Enabled') { - return process.nextTick(next); - } - return s3.send(new PutBucketVersioningCommand({ - Bucket: testBucket, - VersioningConfiguration: { Status: 'Enabled' }, - })) - .then(() => next()) - .catch(next); - }, - next => { - if (versioning !== 'Enabled') { - return process.nextTick(next); - } - return s3.send(new PutBucketVersioningCommand({ - Bucket: emptyBucket, - VersioningConfiguration: { Status: 'Enabled' }, - })) - .then(() => next()) - .catch(next); - }, - next => async.times(3, (n, cb) => { - const keyName = `oldkey${n}`; - s3.send(new PutObjectCommand({ - Bucket: testBucket, - Key: keyName, - Body: '123', - Tagging: 'mykey=myvalue', - })) - .then(data => { - expectedKeyVersions[keyName] = data.VersionId; - cb(); - }) - .catch(cb); - }, next), - next => { - date = new Date(Date.now()).toISOString(); - return async.times(5, (n, cb) => { - const keyName = `key${n}`; - s3.send(new PutObjectCommand({ - Bucket: testBucket, - Key: keyName, - Body: '123', - Tagging: 'mykey=myvalue', - })) - .then(data => { - expectedKeyVersions[keyName] = data.VersionId; - cb(); + before(done => + async.series( + [ + next => { + getCredentials() + .then(creds => { + credentials = creds; + return getS3Hostname(); }) - .catch(cb); - }, next); - }, - ], done)); + .then(() => next()) + .catch(next); + }, + next => + s3 + .send(new CreateBucketCommand({ Bucket: testBucket })) + .then(() => next()) + .catch(next), + next => + s3 + .send(new CreateBucketCommand({ Bucket: emptyBucket })) + .then(() => next()) + .catch(next), + next => { + if (versioning !== 'Enabled') { + return process.nextTick(next); + } + return s3 + .send( + new PutBucketVersioningCommand({ + Bucket: testBucket, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + if (versioning !== 'Enabled') { + return process.nextTick(next); + } + return s3 + .send( + new PutBucketVersioningCommand({ + Bucket: emptyBucket, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ) + .then(() => next()) + .catch(next); + }, + next => + async.times( + 3, + (n, cb) => { + const keyName = `oldkey${n}`; + s3.send( + new PutObjectCommand({ + Bucket: testBucket, + Key: keyName, + Body: '123', + Tagging: 'mykey=myvalue', + }), + ) + .then(data => { + expectedKeyVersions[keyName] = data.VersionId; + cb(); + }) + .catch(cb); + }, + next, + ), + next => { + date = new Date(Date.now()).toISOString(); + return async.times( + 5, + (n, cb) => { + const keyName = `key${n}`; + s3.send( + new PutObjectCommand({ + Bucket: testBucket, + Key: keyName, + Body: '123', + Tagging: 'mykey=myvalue', + }), + ) + .then(data => { + expectedKeyVersions[keyName] = data.VersionId; + cb(); + }) + .catch(cb); + }, + next, + ); + }, + ], + done, + ), + ); after(async () => { await removeAllVersionsPromise({ Bucket: testBucket }); await s3.send(new DeleteBucketCommand({ Bucket: testBucket })); - await s3.send(new DeleteBucketCommand({ Bucket: emptyBucket })); + await s3.send(new DeleteBucketCommand({ Bucket: emptyBucket })); }); it('should return empty list of current versions if bucket is empty', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: emptyBucket, - queryObj: { 'list-type': 'current' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); - - assert.strictEqual(data.IsTruncated, false); - assert(!data.NextMarker); - assert.strictEqual(data.MaxKeys, 1000); - assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); - assert.strictEqual(data.Contents.length, 0); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: emptyBucket, + queryObj: { 'list-type': 'current' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); + + assert.strictEqual(data.IsTruncated, false); + assert(!data.NextMarker); + assert.strictEqual(data.MaxKeys, 1000); + assert.strictEqual( + data.MaxScannedLifecycleListingEntries, + config.maxScannedLifecycleListingEntries, + ); + assert.strictEqual(data.Contents.length, 0); + return done(); + }, + ); }); it('should return empty list of current versions if prefix does not apply', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'current', 'prefix': 'unknown' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); - - assert.strictEqual(data.IsTruncated, false); - assert(!data.NextMarker); - assert.strictEqual(data.MaxKeys, 1000); - assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); - assert.strictEqual(data.Contents.length, 0); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'current', prefix: 'unknown' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); + + assert.strictEqual(data.IsTruncated, false); + assert(!data.NextMarker); + assert.strictEqual(data.MaxKeys, 1000); + assert.strictEqual( + data.MaxScannedLifecycleListingEntries, + config.maxScannedLifecycleListingEntries, + ); + assert.strictEqual(data.Contents.length, 0); + return done(); + }, + ); }); it('should return empty list if max-keys is set to 0', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'current', 'max-keys': '0' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); - - assert.strictEqual(data.IsTruncated, false); - assert(!data.NextMarker); - assert.strictEqual(data.MaxKeys, 0); - assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); - assert.strictEqual(data.Contents.length, 0); - - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'current', 'max-keys': '0' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); + + assert.strictEqual(data.IsTruncated, false); + assert(!data.NextMarker); + assert.strictEqual(data.MaxKeys, 0); + assert.strictEqual( + data.MaxScannedLifecycleListingEntries, + config.maxScannedLifecycleListingEntries, + ); + assert.strictEqual(data.Contents.length, 0); + + return done(); + }, + ); }); it('should return NoSuchBucket error if bucket does not exist', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: 'idonotexist', - queryObj: { 'list-type': 'current' }, - authCredentials: credentials, - }, err => { - assert.strictEqual(err.code, 'NoSuchBucket'); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: 'idonotexist', + queryObj: { 'list-type': 'current' }, + authCredentials: credentials, + }, + err => { + assert.strictEqual(err.code, 'NoSuchBucket'); + return done(); + }, + ); }); it('should return InvalidArgument error if max-keys is invalid', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'current', 'max-keys': 'a' }, - authCredentials: credentials, - }, err => { - assert.strictEqual(err.code, 'InvalidArgument'); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'current', 'max-keys': 'a' }, + authCredentials: credentials, + }, + err => { + assert.strictEqual(err.code, 'InvalidArgument'); + return done(); + }, + ); }); it('should return InvalidArgument error if max-scanned-lifecycle-listing-entries is invalid', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'current', 'max-scanned-lifecycle-listing-entries': 'a' }, - authCredentials: credentials, - }, err => { - assert.strictEqual(err.code, 'InvalidArgument'); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'current', 'max-scanned-lifecycle-listing-entries': 'a' }, + authCredentials: credentials, + }, + err => { + assert.strictEqual(err.code, 'InvalidArgument'); + return done(); + }, + ); }); it('should return InvalidArgument error if max-scanned-lifecycle-listing-entries is set to 0', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'current', 'max-scanned-lifecycle-listing-entries': '0' }, - authCredentials: credentials, - }, err => { - assert.strictEqual(err.code, 'InvalidArgument'); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'current', 'max-scanned-lifecycle-listing-entries': '0' }, + authCredentials: credentials, + }, + err => { + assert.strictEqual(err.code, 'InvalidArgument'); + return done(); + }, + ); }); it('should return InvalidArgument if max-scanned-lifecycle-listing-entries exceeds the default value', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'current', 'max-scanned-lifecycle-listing-entries': - (config.maxScannedLifecycleListingEntries + 1).toString() }, - authCredentials: credentials, - }, err => { - assert.strictEqual(err.code, 'InvalidArgument'); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { + 'list-type': 'current', + 'max-scanned-lifecycle-listing-entries': ( + config.maxScannedLifecycleListingEntries + 1 + ).toString(), + }, + authCredentials: credentials, + }, + err => { + assert.strictEqual(err.code, 'InvalidArgument'); + return done(); + }, + ); }); it('should return all the current versions', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'current' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); - - assert.strictEqual(data.IsTruncated, false); - assert(!data.NextMarker); - assert.strictEqual(data.MaxKeys, 1000); - assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); - - const contents = data.Contents; - assert.strictEqual(contents.length, 8); - checkContents(contents, expectedKeyVersions); - - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'current' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); + + assert.strictEqual(data.IsTruncated, false); + assert(!data.NextMarker); + assert.strictEqual(data.MaxKeys, 1000); + assert.strictEqual( + data.MaxScannedLifecycleListingEntries, + config.maxScannedLifecycleListingEntries, + ); + + const contents = data.Contents; + assert.strictEqual(contents.length, 8); + checkContents(contents, expectedKeyVersions); + + return done(); + }, + ); }); it('should return all the current versions before max scanned entries value is reached', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'current', 'max-scanned-lifecycle-listing-entries': '5' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'current', 'max-scanned-lifecycle-listing-entries': '5' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); - assert.strictEqual(data.IsTruncated, true); - assert.strictEqual(data.NextMarker, 'key4'); - assert.strictEqual(data.MaxKeys, 1000); - assert.strictEqual(data.MaxScannedLifecycleListingEntries, 5); + assert.strictEqual(data.IsTruncated, true); + assert.strictEqual(data.NextMarker, 'key4'); + assert.strictEqual(data.MaxKeys, 1000); + assert.strictEqual(data.MaxScannedLifecycleListingEntries, 5); - const contents = data.Contents; - assert.strictEqual(contents.length, 5); - checkContents(contents, expectedKeyVersions); + const contents = data.Contents; + assert.strictEqual(contents.length, 5); + checkContents(contents, expectedKeyVersions); - return done(); - }); + return done(); + }, + ); }); it('should return all the current versions with prefix old', done => { const prefix = 'old'; - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'current', prefix }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'current', prefix }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); + + assert.strictEqual(data.IsTruncated, false); + assert(!data.NextMarker); + assert.strictEqual(data.MaxKeys, 1000); + assert.strictEqual( + data.MaxScannedLifecycleListingEntries, + config.maxScannedLifecycleListingEntries, + ); + assert.strictEqual(data.Prefix, prefix); + + const contents = data.Contents; + assert.strictEqual(contents.length, 3); + checkContents(contents, expectedKeyVersions); + + return done(); + }, + ); + }); - assert.strictEqual(data.IsTruncated, false); - assert(!data.NextMarker); - assert.strictEqual(data.MaxKeys, 1000); - assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); - assert.strictEqual(data.Prefix, prefix); + it('should return the current versions before a defined date', done => { + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'current', 'before-date': date }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); + + assert.strictEqual(data.IsTruncated, false); + assert(!data.NextMarker); + assert.strictEqual(data.MaxKeys, 1000); + assert.strictEqual( + data.MaxScannedLifecycleListingEntries, + config.maxScannedLifecycleListingEntries, + ); + assert.strictEqual(data.Contents.length, 3); + assert.strictEqual(data.BeforeDate, date); + + const contents = data.Contents; + checkContents(contents, expectedKeyVersions); + assert.strictEqual(contents[0].Key, 'oldkey0'); + assert.strictEqual(contents[1].Key, 'oldkey1'); + assert.strictEqual(contents[2].Key, 'oldkey2'); + return done(); + }, + ); + }); - const contents = data.Contents; - assert.strictEqual(contents.length, 3); - checkContents(contents, expectedKeyVersions); + it('should truncate list of current versions before a defined date', done => { + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'current', 'before-date': date, 'max-keys': '1' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); + + assert.strictEqual(data.IsTruncated, true); + assert.strictEqual(data.NextMarker, 'oldkey0'); + assert.strictEqual(data.MaxKeys, 1); + assert.strictEqual( + data.MaxScannedLifecycleListingEntries, + config.maxScannedLifecycleListingEntries, + ); + assert.strictEqual(data.BeforeDate, date); + assert.strictEqual(data.Contents.length, 1); + + const contents = data.Contents; + checkContents(contents, expectedKeyVersions); + assert.strictEqual(contents[0].Key, 'oldkey0'); + return done(); + }, + ); + }); - return done(); - }); + it('should return the next truncate list of current versions before a defined date', done => { + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'current', 'before-date': date, 'max-keys': '1', marker: 'oldkey0' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); + + assert.strictEqual(data.IsTruncated, true); + assert.strictEqual(data.Marker, 'oldkey0'); + assert.strictEqual(data.NextMarker, 'oldkey1'); + assert.strictEqual(data.MaxKeys, 1); + assert.strictEqual( + data.MaxScannedLifecycleListingEntries, + config.maxScannedLifecycleListingEntries, + ); + assert.strictEqual(data.Contents.length, 1); + + const contents = data.Contents; + checkContents(contents, expectedKeyVersions); + assert.strictEqual(contents[0].Key, 'oldkey1'); + assert.strictEqual(data.BeforeDate, date); + return done(); + }, + ); }); - it('should return the current versions before a defined date', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'current', 'before-date': date }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); + it('should return the last truncate list of current versions before a defined date', done => { + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'current', 'before-date': date, 'max-keys': '1', marker: 'oldkey1' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); + + assert.strictEqual(data.IsTruncated, false); + assert.strictEqual(data.MaxKeys, 1); + assert.strictEqual( + data.MaxScannedLifecycleListingEntries, + config.maxScannedLifecycleListingEntries, + ); + assert.strictEqual(data.Marker, 'oldkey1'); + assert.strictEqual(data.BeforeDate, date); + + const contents = data.Contents; + assert.strictEqual(contents.length, 1); + checkContents(contents, expectedKeyVersions); + assert.strictEqual(contents[0].Key, 'oldkey2'); + return done(); + }, + ); + }); + }); +}); - assert.strictEqual(data.IsTruncated, false); - assert(!data.NextMarker); - assert.strictEqual(data.MaxKeys, 1000); - assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); - assert.strictEqual(data.Contents.length, 3); - assert.strictEqual(data.BeforeDate, date); +describe('listLifecycleCurrents with bucket versioning enabled and maxKeys', () => { + const testBucket = 'bucket-for-list-lifecycle-current-tests-truncated'; + const expectedKeyVersions = {}; - const contents = data.Contents; - checkContents(contents, expectedKeyVersions); - assert.strictEqual(contents[0].Key, 'oldkey0'); - assert.strictEqual(contents[1].Key, 'oldkey1'); - assert.strictEqual(contents[2].Key, 'oldkey2'); - return done(); - }); - }); + before(done => + async.series( + [ + next => { + getCredentials() + .then(creds => { + credentials = creds; + return getS3Hostname(); + }) + .then(() => next()) + .catch(next); + }, + next => + s3 + .send(new CreateBucketCommand({ Bucket: testBucket })) + .then(() => next()) + .catch(next), + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: testBucket, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ) + .then(() => next()) + .catch(next), + next => + async.times( + 3, + (n, cb) => { + const keyName = 'key0'; + s3.send( + new PutObjectCommand({ + Bucket: testBucket, + Key: keyName, + Body: '123', + Tagging: 'mykey=myvalue', + }), + ) + .then(data => { + expectedKeyVersions[keyName] = data.VersionId; + cb(); + }) + .catch(err => cb(err)); + }, + next, + ), + next => + async.times( + 5, + (n, cb) => { + const keyName = 'key1'; + s3.send( + new PutObjectCommand({ + Bucket: testBucket, + Key: keyName, + Body: '123', + Tagging: 'mykey=myvalue', + }), + ) + .then(data => { + expectedKeyVersions[keyName] = data.VersionId; + cb(); + }) + .catch(err => cb(err)); + }, + next, + ), + next => + async.times( + 3, + (n, cb) => { + const keyName = 'key2'; + s3.send( + new PutObjectCommand({ + Bucket: testBucket, + Key: keyName, + Body: '123', + Tagging: 'mykey=myvalue', + }), + ) + .then(data => { + expectedKeyVersions[keyName] = data.VersionId; + cb(); + }) + .catch(err => cb(err)); + }, + next, + ), + ], + done, + ), + ); - it('should truncate list of current versions before a defined date', done => { - makeBackbeatRequest({ + after(async () => { + await removeAllVersionsPromise({ Bucket: testBucket }); + await s3.send(new DeleteBucketCommand({ Bucket: testBucket })); + }); + + it('should return truncated lists - part 1', done => { + makeBackbeatRequest( + { method: 'GET', bucket: testBucket, - queryObj: { 'list-type': 'current', 'before-date': date, 'max-keys': '1' }, + queryObj: { 'list-type': 'current', 'max-keys': '1' }, authCredentials: credentials, - }, (err, response) => { + }, + (err, response) => { assert.ifError(err); assert.strictEqual(response.statusCode, 200); const data = JSON.parse(response.body); assert.strictEqual(data.IsTruncated, true); - assert.strictEqual(data.NextMarker, 'oldkey0'); + assert.strictEqual(data.NextMarker, 'key0'); assert.strictEqual(data.MaxKeys, 1); assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); - assert.strictEqual(data.BeforeDate, date); assert.strictEqual(data.Contents.length, 1); const contents = data.Contents; + assert.strictEqual(contents.length, 1); checkContents(contents, expectedKeyVersions); - assert.strictEqual(contents[0].Key, 'oldkey0'); + assert.strictEqual(contents[0].Key, 'key0'); return done(); - }); - }); + }, + ); + }); - it('should return the next truncate list of current versions before a defined date', done => { - makeBackbeatRequest({ + it('should return truncated lists - part 2', done => { + makeBackbeatRequest( + { method: 'GET', bucket: testBucket, - queryObj: { 'list-type': 'current', 'before-date': date, 'max-keys': '1', 'marker': 'oldkey0' }, + queryObj: { + 'list-type': 'current', + 'max-keys': '1', + marker: 'key0', + }, authCredentials: credentials, - }, (err, response) => { + }, + (err, response) => { assert.ifError(err); assert.strictEqual(response.statusCode, 200); const data = JSON.parse(response.body); + assert.strictEqual(data.Marker, 'key0'); assert.strictEqual(data.IsTruncated, true); - assert.strictEqual(data.Marker, 'oldkey0'); - assert.strictEqual(data.NextMarker, 'oldkey1'); + assert.strictEqual(data.NextMarker, 'key1'); assert.strictEqual(data.MaxKeys, 1); assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); assert.strictEqual(data.Contents.length, 1); const contents = data.Contents; + assert.strictEqual(contents.length, 1); checkContents(contents, expectedKeyVersions); - assert.strictEqual(contents[0].Key, 'oldkey1'); - assert.strictEqual(data.BeforeDate, date); + assert.strictEqual(contents[0].Key, 'key1'); return done(); - }); - }); + }, + ); + }); - it('should return the last truncate list of current versions before a defined date', done => { - makeBackbeatRequest({ + it('should return truncated lists - part 3', done => { + makeBackbeatRequest( + { method: 'GET', bucket: testBucket, - queryObj: { 'list-type': 'current', 'before-date': date, 'max-keys': '1', 'marker': 'oldkey1' }, + queryObj: { + 'list-type': 'current', + 'max-keys': '1', + marker: 'key1', + }, authCredentials: credentials, - }, (err, response) => { + }, + (err, response) => { assert.ifError(err); assert.strictEqual(response.statusCode, 200); const data = JSON.parse(response.body); + assert(!data.NextMarker); assert.strictEqual(data.IsTruncated, false); + assert.strictEqual(data.Marker, 'key1'); assert.strictEqual(data.MaxKeys, 1); assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); - assert.strictEqual(data.Marker, 'oldkey1'); - assert.strictEqual(data.BeforeDate, date); + assert.strictEqual(data.Contents.length, 1); const contents = data.Contents; assert.strictEqual(contents.length, 1); checkContents(contents, expectedKeyVersions); - assert.strictEqual(contents[0].Key, 'oldkey2'); + assert.strictEqual(contents[0].Key, 'key2'); return done(); - }); - }); - }); -}); - -describe('listLifecycleCurrents with bucket versioning enabled and maxKeys', () => { - const testBucket = 'bucket-for-list-lifecycle-current-tests-truncated'; - const expectedKeyVersions = {}; - - before(done => async.series([ - next => { - getCredentials() - .then(creds => { - credentials = creds; - return getS3Hostname(); - }) - .then(() => next()) - .catch(next); - }, - next => s3.send(new CreateBucketCommand({ Bucket: testBucket })) - .then(() => next()) - .catch(next), - next => s3.send(new PutBucketVersioningCommand({ - Bucket: testBucket, - VersioningConfiguration: { Status: 'Enabled' }, - })) - .then(() => next()) - .catch(next), - next => async.times(3, (n, cb) => { - const keyName = 'key0'; - s3.send(new PutObjectCommand({ - Bucket: testBucket, - Key: keyName, - Body: '123', - Tagging: 'mykey=myvalue', - })) - .then(data => { - expectedKeyVersions[keyName] = data.VersionId; - cb(); - }) - .catch(err => cb(err)); - }, next), - next => async.times(5, (n, cb) => { - const keyName = 'key1'; - s3.send(new PutObjectCommand({ - Bucket: testBucket, - Key: keyName, - Body: '123', - Tagging: 'mykey=myvalue', - })) - .then(data => { - expectedKeyVersions[keyName] = data.VersionId; - cb(); - }) - .catch(err => cb(err)); - }, next), - next => async.times(3, (n, cb) => { - const keyName = 'key2'; - s3.send(new PutObjectCommand({ - Bucket: testBucket, - Key: keyName, - Body: '123', - Tagging: 'mykey=myvalue', - })) - .then(data => { - expectedKeyVersions[keyName] = data.VersionId; - cb(); - }) - .catch(err => cb(err)); - }, next), - ], done)); - - after(async () => { - await removeAllVersionsPromise({ Bucket: testBucket }); - await s3.send(new DeleteBucketCommand({ Bucket: testBucket })); - }); - - it('should return truncated lists - part 1', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'current', 'max-keys': '1' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); - - assert.strictEqual(data.IsTruncated, true); - assert.strictEqual(data.NextMarker, 'key0'); - assert.strictEqual(data.MaxKeys, 1); - assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); - assert.strictEqual(data.Contents.length, 1); - - const contents = data.Contents; - assert.strictEqual(contents.length, 1); - checkContents(contents, expectedKeyVersions); - assert.strictEqual(contents[0].Key, 'key0'); - return done(); - }); - }); - - it('should return truncated lists - part 2', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { - 'list-type': 'current', - 'max-keys': '1', - 'marker': 'key0', - }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); - - assert.strictEqual(data.Marker, 'key0'); - assert.strictEqual(data.IsTruncated, true); - assert.strictEqual(data.NextMarker, 'key1'); - assert.strictEqual(data.MaxKeys, 1); - assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); - assert.strictEqual(data.Contents.length, 1); - - const contents = data.Contents; - assert.strictEqual(contents.length, 1); - checkContents(contents, expectedKeyVersions); - assert.strictEqual(contents[0].Key, 'key1'); - return done(); - }); - }); - - it('should return truncated lists - part 3', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { - 'list-type': 'current', - 'max-keys': '1', - 'marker': 'key1', }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); - - assert(!data.NextMarker); - assert.strictEqual(data.IsTruncated, false); - assert.strictEqual(data.Marker, 'key1'); - assert.strictEqual(data.MaxKeys, 1); - assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); - assert.strictEqual(data.Contents.length, 1); - - const contents = data.Contents; - assert.strictEqual(contents.length, 1); - checkContents(contents, expectedKeyVersions); - assert.strictEqual(contents[0].Key, 'key2'); - return done(); - }); + ); }); }); @@ -615,74 +762,107 @@ describe('listLifecycleCurrents with bucket versioning enabled and delete object const keyName2 = 'key2'; const expectedKeyVersions = {}; - before(done => async.series([ - next => { - getCredentials() - .then(creds => { - credentials = creds; - return getS3Hostname(); - }) - .then(() => next()) - .catch(next); - }, - next => s3.send(new CreateBucketCommand({ Bucket: testBucket })) - .then(() => next()) - .catch(next), - next => s3.send(new PutBucketVersioningCommand({ - Bucket: testBucket, - VersioningConfiguration: { Status: 'Enabled' }, - })) - .then(() => next()) - .catch(next), - next => s3.send(new PutObjectCommand({ - Bucket: testBucket, - Key: keyName0, - Body: '123', - Tagging: 'mykey=myvalue', - })) - .then(data => { - expectedKeyVersions[keyName0] = data.VersionId; - next(); - }) - .catch(next), - next => s3.send(new PutObjectCommand({ - Bucket: testBucket, - Key: keyName1, - Body: '123', - Tagging: 'mykey=myvalue', - })) - .then(() => next()) - .catch(next), - next => s3.send(new DeleteObjectCommand({ Bucket: testBucket, Key: keyName1 })) - .then(() => next()) - .catch(next), - next => s3.send(new PutObjectCommand({ - Bucket: testBucket, - Key: keyName2, - Body: '123', - Tagging: 'mykey=myvalue', - })) - .then(data => { - expectedKeyVersions[keyName2] = data.VersionId; - next(); - }) - .catch(next), - next => s3.send(new PutObjectCommand({ - Bucket: testBucket, - Key: keyName2, - Body: '123', - Tagging: 'mykey=myvalue', - })) - .then(data => s3.send(new DeleteObjectCommand({ - Bucket: testBucket, - Key: keyName2, - VersionId: data.VersionId, - })) - .then(() => next()) - .catch(next)) - .catch(next), - ], done)); - + before(done => + async.series( + [ + next => { + getCredentials() + .then(creds => { + credentials = creds; + return getS3Hostname(); + }) + .then(() => next()) + .catch(next); + }, + next => + s3 + .send(new CreateBucketCommand({ Bucket: testBucket })) + .then(() => next()) + .catch(next), + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: testBucket, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ) + .then(() => next()) + .catch(next), + next => + s3 + .send( + new PutObjectCommand({ + Bucket: testBucket, + Key: keyName0, + Body: '123', + Tagging: 'mykey=myvalue', + }), + ) + .then(data => { + expectedKeyVersions[keyName0] = data.VersionId; + next(); + }) + .catch(next), + next => + s3 + .send( + new PutObjectCommand({ + Bucket: testBucket, + Key: keyName1, + Body: '123', + Tagging: 'mykey=myvalue', + }), + ) + .then(() => next()) + .catch(next), + next => + s3 + .send(new DeleteObjectCommand({ Bucket: testBucket, Key: keyName1 })) + .then(() => next()) + .catch(next), + next => + s3 + .send( + new PutObjectCommand({ + Bucket: testBucket, + Key: keyName2, + Body: '123', + Tagging: 'mykey=myvalue', + }), + ) + .then(data => { + expectedKeyVersions[keyName2] = data.VersionId; + next(); + }) + .catch(next), + next => + s3 + .send( + new PutObjectCommand({ + Bucket: testBucket, + Key: keyName2, + Body: '123', + Tagging: 'mykey=myvalue', + }), + ) + .then(data => + s3 + .send( + new DeleteObjectCommand({ + Bucket: testBucket, + Key: keyName2, + VersionId: data.VersionId, + }), + ) + .then(() => next()) + .catch(next), + ) + .catch(next), + ], + done, + ), + ); after(async () => { await removeAllVersionsPromise({ Bucket: testBucket }); @@ -690,57 +870,63 @@ describe('listLifecycleCurrents with bucket versioning enabled and delete object }); it('should return truncated lists - part 1', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'current', 'max-keys': '1' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); - - assert.strictEqual(data.IsTruncated, true); - assert.strictEqual(data.NextMarker, keyName0); - assert.strictEqual(data.MaxKeys, 1); - assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); - assert.strictEqual(data.Contents.length, 1); - - const contents = data.Contents; - assert.strictEqual(contents.length, 1); - checkContents(contents, expectedKeyVersions); - assert.strictEqual(contents[0].Key, keyName0); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'current', 'max-keys': '1' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); + + assert.strictEqual(data.IsTruncated, true); + assert.strictEqual(data.NextMarker, keyName0); + assert.strictEqual(data.MaxKeys, 1); + assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); + assert.strictEqual(data.Contents.length, 1); + + const contents = data.Contents; + assert.strictEqual(contents.length, 1); + checkContents(contents, expectedKeyVersions); + assert.strictEqual(contents[0].Key, keyName0); + return done(); + }, + ); }); it('should return truncated lists - part 2', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { - 'list-type': 'current', - 'max-keys': '1', - 'marker': keyName0, + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { + 'list-type': 'current', + 'max-keys': '1', + marker: keyName0, + }, + authCredentials: credentials, }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); - - assert(!data.NextMarker); - assert.strictEqual(data.IsTruncated, false); - assert.strictEqual(data.Marker, keyName0); - assert.strictEqual(data.MaxKeys, 1); - assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); - assert.strictEqual(data.Contents.length, 1); - - const contents = data.Contents; - assert.strictEqual(contents.length, 1); - checkContents(contents, expectedKeyVersions); - assert.strictEqual(contents[0].Key, keyName2); - return done(); - }); + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); + + assert(!data.NextMarker); + assert.strictEqual(data.IsTruncated, false); + assert.strictEqual(data.Marker, keyName0); + assert.strictEqual(data.MaxKeys, 1); + assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); + assert.strictEqual(data.Contents.length, 1); + + const contents = data.Contents; + assert.strictEqual(contents.length, 1); + checkContents(contents, expectedKeyVersions); + assert.strictEqual(contents[0].Key, keyName2); + return done(); + }, + ); }); }); diff --git a/tests/functional/backbeat/listLifecycleOrphanDeleteMarkers.js b/tests/functional/backbeat/listLifecycleOrphanDeleteMarkers.js index 7eb9608642..5e08d2d642 100644 --- a/tests/functional/backbeat/listLifecycleOrphanDeleteMarkers.js +++ b/tests/functional/backbeat/listLifecycleOrphanDeleteMarkers.js @@ -49,84 +49,135 @@ function checkContents(contents) { } function createDeleteMarker(s3, bucketName, keyName, cb) { - return async.series([ - next => s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: keyName, - Body: '123', - Tagging: 'mykey=myvalue' - })) - .then(() => next()) - .catch(next), - next => s3.send(new DeleteObjectCommand({ Bucket: bucketName, Key: keyName })) - .then(() => next()) - .catch(next), - ], cb); + return async.series( + [ + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucketName, + Key: keyName, + Body: '123', + Tagging: 'mykey=myvalue', + }), + ) + .then(() => next()) + .catch(next), + next => + s3 + .send(new DeleteObjectCommand({ Bucket: bucketName, Key: keyName })) + .then(() => next()) + .catch(next), + ], + cb, + ); } function createOrphanDeleteMarker(s3, bucketName, keyName, cb) { let versionId; - return async.series([ - next => s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: keyName, - Body: '123', - Tagging: 'mykey=myvalue' - })) - .then(data => { - versionId = data.VersionId; - next(); - }) - .catch(next), - next => s3.send(new DeleteObjectCommand({ Bucket: bucketName, Key: keyName })) - .then(() => next()) - .catch(next), - next => s3.send(new DeleteObjectCommand({ Bucket: bucketName, Key: keyName, VersionId: versionId })) - .then(() => next()) - .catch(next), - ], cb); + return async.series( + [ + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucketName, + Key: keyName, + Body: '123', + Tagging: 'mykey=myvalue', + }), + ) + .then(data => { + versionId = data.VersionId; + next(); + }) + .catch(next), + next => + s3 + .send(new DeleteObjectCommand({ Bucket: bucketName, Key: keyName })) + .then(() => next()) + .catch(next), + next => + s3 + .send(new DeleteObjectCommand({ Bucket: bucketName, Key: keyName, VersionId: versionId })) + .then(() => next()) + .catch(next), + ], + cb, + ); } describe('listLifecycleOrphanDeleteMarkers', () => { let date; - before(done => async.series([ - next => getCredentials().then(creds => { - credentials = creds; - next(); - }).catch(next), - next => s3.send(new CreateBucketCommand({ Bucket: testBucket })) - .then(() => next()) - .catch(next), - next => s3.send(new CreateBucketCommand({ Bucket: emptyBucket })) - .then(() => next()) - .catch(next), - next => s3.send(new CreateBucketCommand({ Bucket: nonVersionedBucket })) - .then(() => next()) - .catch(next), - next => s3.send(new PutBucketVersioningCommand({ - Bucket: testBucket, - VersioningConfiguration: { Status: 'Enabled' }, - })) - .then(() => next()) - .catch(next), - next => s3.send(new PutBucketVersioningCommand({ - Bucket: emptyBucket, - VersioningConfiguration: { Status: 'Enabled' }, - })) - .then(() => next()) - .catch(next), - next => async.times(3, (n, cb) => { - createOrphanDeleteMarker(s3, testBucket, `key${n}old`, cb); - }, next), - next => createDeleteMarker(s3, testBucket, 'no-orphan-delete-marker', next), - next => { - date = new Date(Date.now()).toISOString(); - return async.times(5, (n, cb) => { - createOrphanDeleteMarker(s3, testBucket, `key${n}`, cb); - }, next); - }, - ], done)); + before(done => + async.series( + [ + next => + getCredentials() + .then(creds => { + credentials = creds; + next(); + }) + .catch(next), + next => + s3 + .send(new CreateBucketCommand({ Bucket: testBucket })) + .then(() => next()) + .catch(next), + next => + s3 + .send(new CreateBucketCommand({ Bucket: emptyBucket })) + .then(() => next()) + .catch(next), + next => + s3 + .send(new CreateBucketCommand({ Bucket: nonVersionedBucket })) + .then(() => next()) + .catch(next), + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: testBucket, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ) + .then(() => next()) + .catch(next), + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: emptyBucket, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ) + .then(() => next()) + .catch(next), + next => + async.times( + 3, + (n, cb) => { + createOrphanDeleteMarker(s3, testBucket, `key${n}old`, cb); + }, + next, + ), + next => createDeleteMarker(s3, testBucket, 'no-orphan-delete-marker', next), + next => { + date = new Date(Date.now()).toISOString(); + return async.times( + 5, + (n, cb) => { + createOrphanDeleteMarker(s3, testBucket, `key${n}`, cb); + }, + next, + ); + }, + ], + done, + ), + ); after(async () => { await removeAllVersionsPromise({ Bucket: testBucket }); @@ -136,409 +187,465 @@ describe('listLifecycleOrphanDeleteMarkers', () => { }); it('should return empty list of orphan delete markers if bucket is empty', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: emptyBucket, - queryObj: { 'list-type': 'orphan' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - - const data = JSON.parse(response.body); - assert.strictEqual(data.IsTruncated, false); - assert(!data.NextMarker); - assert.strictEqual(data.MaxKeys, 1000); - assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); - assert.strictEqual(data.Contents.length, 0); - - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: emptyBucket, + queryObj: { 'list-type': 'orphan' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + + const data = JSON.parse(response.body); + assert.strictEqual(data.IsTruncated, false); + assert(!data.NextMarker); + assert.strictEqual(data.MaxKeys, 1000); + assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); + assert.strictEqual(data.Contents.length, 0); + + return done(); + }, + ); }); it('should return empty list of orphan delete markers if prefix does not apply', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'orphan', 'prefix': 'unknown' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - - const data = JSON.parse(response.body); - assert.strictEqual(data.IsTruncated, false); - assert(!data.NextMarker); - assert.strictEqual(data.MaxKeys, 1000); - assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); - assert.strictEqual(data.Contents.length, 0); - - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'orphan', prefix: 'unknown' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + + const data = JSON.parse(response.body); + assert.strictEqual(data.IsTruncated, false); + assert(!data.NextMarker); + assert.strictEqual(data.MaxKeys, 1000); + assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); + assert.strictEqual(data.Contents.length, 0); + + return done(); + }, + ); }); it('should return empty list if max-keys is set to 0', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'orphan', 'max-keys': '0' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - - const data = JSON.parse(response.body); - assert.strictEqual(data.IsTruncated, false); - assert(!data.NextMarker); - assert.strictEqual(data.MaxKeys, 0); - assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); - assert.strictEqual(data.Contents.length, 0); - - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'orphan', 'max-keys': '0' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + + const data = JSON.parse(response.body); + assert.strictEqual(data.IsTruncated, false); + assert(!data.NextMarker); + assert.strictEqual(data.MaxKeys, 0); + assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); + assert.strictEqual(data.Contents.length, 0); + + return done(); + }, + ); }); it('should return InvalidArgument error if max-keys is invalid', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'orphan', 'max-keys': 'a' }, - authCredentials: credentials, - }, err => { - assert.strictEqual(err.code, 'InvalidArgument'); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'orphan', 'max-keys': 'a' }, + authCredentials: credentials, + }, + err => { + assert.strictEqual(err.code, 'InvalidArgument'); + return done(); + }, + ); }); it('should return InvalidArgument error if max-scanned-lifecycle-listing-entries is invalid', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'orphan', 'max-scanned-lifecycle-listing-entries': 'a' }, - authCredentials: credentials, - }, err => { - assert.strictEqual(err.code, 'InvalidArgument'); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'orphan', 'max-scanned-lifecycle-listing-entries': 'a' }, + authCredentials: credentials, + }, + err => { + assert.strictEqual(err.code, 'InvalidArgument'); + return done(); + }, + ); }); it('should return InvalidArgument error if max-scanned-lifecycle-listing-entries is set to 0', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'orphan', 'max-scanned-lifecycle-listing-entries': '0' }, - authCredentials: credentials, - }, err => { - assert.strictEqual(err.code, 'InvalidArgument'); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'orphan', 'max-scanned-lifecycle-listing-entries': '0' }, + authCredentials: credentials, + }, + err => { + assert.strictEqual(err.code, 'InvalidArgument'); + return done(); + }, + ); }); it('should return InvalidArgument error if max-scanned-lifecycle-listing-entries is set to 2', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'orphan', 'max-scanned-lifecycle-listing-entries': '2' }, - authCredentials: credentials, - }, err => { - assert.strictEqual(err.code, 'InvalidArgument'); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'orphan', 'max-scanned-lifecycle-listing-entries': '2' }, + authCredentials: credentials, + }, + err => { + assert.strictEqual(err.code, 'InvalidArgument'); + return done(); + }, + ); }); it('should return InvalidArgument if max-scanned-lifecycle-listing-entries exceeds the default value', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'orphan', 'max-scanned-lifecycle-listing-entries': - (config.maxScannedLifecycleListingEntries + 1).toString() }, - authCredentials: credentials, - }, err => { - assert.strictEqual(err.code, 'InvalidArgument'); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { + 'list-type': 'orphan', + 'max-scanned-lifecycle-listing-entries': (config.maxScannedLifecycleListingEntries + 1).toString(), + }, + authCredentials: credentials, + }, + err => { + assert.strictEqual(err.code, 'InvalidArgument'); + return done(); + }, + ); }); it('should return error if bucket does not exist', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: 'idonotexist', - queryObj: { 'list-type': 'orphan' }, - authCredentials: credentials, - }, err => { - assert.strictEqual(err.code, 'NoSuchBucket'); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: 'idonotexist', + queryObj: { 'list-type': 'orphan' }, + authCredentials: credentials, + }, + err => { + assert.strictEqual(err.code, 'NoSuchBucket'); + return done(); + }, + ); }); it('should return all the orphan delete markers', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'orphan' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - - const data = JSON.parse(response.body); - assert.strictEqual(data.IsTruncated, false); - assert(!data.NextMarker); - assert.strictEqual(data.MaxKeys, 1000); - assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); - - const contents = data.Contents; - assert.strictEqual(contents.length, 8); - checkContents(contents); - - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'orphan' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + + const data = JSON.parse(response.body); + assert.strictEqual(data.IsTruncated, false); + assert(!data.NextMarker); + assert.strictEqual(data.MaxKeys, 1000); + assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); + + const contents = data.Contents; + assert.strictEqual(contents.length, 8); + checkContents(contents); + + return done(); + }, + ); }); it('should only return delete marker that passed the full keys evaluation to prevent false positives', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'orphan', 'max-scanned-lifecycle-listing-entries': '4' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - - const data = JSON.parse(response.body); - assert.strictEqual(data.IsTruncated, true); - assert.strictEqual(data.MaxKeys, 1000); - assert.strictEqual(data.MaxScannedLifecycleListingEntries, 4); - - // Depending on the metadata bucket key format, the orphan delete marker is denoted by 1 or 2 entries, - // which results in a difference in the number of keys scanned and, consequently, - // affects the value of the NextMarker and Contents. - const contents = data.Contents; - const nextMarker = data.NextMarker; - - if (process.env.DEFAULT_BUCKET_KEY_FORMAT === 'v1') { - // With v1 metadata bucket key format, master key is automaticaly deleted - // when the last version of an object is a delete marker - assert.strictEqual(nextMarker, 'key1'); - assert.strictEqual(contents.length, 3); - assert.strictEqual(contents[0].Key, 'key0'); - assert.strictEqual(contents[1].Key, 'key0old'); - assert.strictEqual(contents[2].Key, 'key1'); - } else { - assert.strictEqual(nextMarker, 'key0'); - assert.strictEqual(contents.length, 1); - assert.strictEqual(contents[0].Key, 'key0'); - } - checkContents(contents); - - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'orphan', 'max-scanned-lifecycle-listing-entries': '4' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + + const data = JSON.parse(response.body); + assert.strictEqual(data.IsTruncated, true); + assert.strictEqual(data.MaxKeys, 1000); + assert.strictEqual(data.MaxScannedLifecycleListingEntries, 4); + + // Depending on the metadata bucket key format, the orphan delete marker is denoted by 1 or 2 entries, + // which results in a difference in the number of keys scanned and, consequently, + // affects the value of the NextMarker and Contents. + const contents = data.Contents; + const nextMarker = data.NextMarker; + + if (process.env.DEFAULT_BUCKET_KEY_FORMAT === 'v1') { + // With v1 metadata bucket key format, master key is automaticaly deleted + // when the last version of an object is a delete marker + assert.strictEqual(nextMarker, 'key1'); + assert.strictEqual(contents.length, 3); + assert.strictEqual(contents[0].Key, 'key0'); + assert.strictEqual(contents[1].Key, 'key0old'); + assert.strictEqual(contents[2].Key, 'key1'); + } else { + assert.strictEqual(nextMarker, 'key0'); + assert.strictEqual(contents.length, 1); + assert.strictEqual(contents[0].Key, 'key0'); + } + checkContents(contents); + + return done(); + }, + ); }); it('should return all the orphan delete markers before max scanned entries value is reached', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'orphan', 'max-scanned-lifecycle-listing-entries': '3' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - - const data = JSON.parse(response.body); - assert.strictEqual(data.IsTruncated, true); - assert.strictEqual(data.MaxKeys, 1000); - assert.strictEqual(data.MaxScannedLifecycleListingEntries, 3); - - // Depending on the metadata bucket key format, the orphan delete marker is denoted by 1 or 2 entries, - // which results in a difference in the number of keys scanned and, consequently, - // affects the value of the NextMarker and Contents. - const contents = data.Contents; - const nextMarker = data.NextMarker; - - if (process.env.DEFAULT_BUCKET_KEY_FORMAT === 'v1') { - // With v1 metadata bucket key format, master key is automaticaly deleted - // when the last version of an object is a delete marker - assert.strictEqual(nextMarker, 'key0old'); - assert.strictEqual(contents.length, 2); - assert.strictEqual(contents[0].Key, 'key0'); - assert.strictEqual(contents[1].Key, 'key0old'); - } else { - assert.strictEqual(nextMarker, 'key0'); - assert.strictEqual(contents.length, 1); - assert.strictEqual(contents[0].Key, 'key0'); - } - checkContents(contents); - - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'orphan', 'max-scanned-lifecycle-listing-entries': '3' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + + const data = JSON.parse(response.body); + assert.strictEqual(data.IsTruncated, true); + assert.strictEqual(data.MaxKeys, 1000); + assert.strictEqual(data.MaxScannedLifecycleListingEntries, 3); + + // Depending on the metadata bucket key format, the orphan delete marker is denoted by 1 or 2 entries, + // which results in a difference in the number of keys scanned and, consequently, + // affects the value of the NextMarker and Contents. + const contents = data.Contents; + const nextMarker = data.NextMarker; + + if (process.env.DEFAULT_BUCKET_KEY_FORMAT === 'v1') { + // With v1 metadata bucket key format, master key is automaticaly deleted + // when the last version of an object is a delete marker + assert.strictEqual(nextMarker, 'key0old'); + assert.strictEqual(contents.length, 2); + assert.strictEqual(contents[0].Key, 'key0'); + assert.strictEqual(contents[1].Key, 'key0old'); + } else { + assert.strictEqual(nextMarker, 'key0'); + assert.strictEqual(contents.length, 1); + assert.strictEqual(contents[0].Key, 'key0'); + } + checkContents(contents); + + return done(); + }, + ); }); it('should return all the orphan delete markers with prefix key1', done => { const prefix = 'key1'; - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'orphan', prefix }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - - const data = JSON.parse(response.body); - assert.strictEqual(data.IsTruncated, false); - assert(!data.NextMarker); - assert.strictEqual(data.MaxKeys, 1000); - assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); - assert.strictEqual(data.Prefix, prefix); - - const contents = data.Contents; - assert.strictEqual(contents.length, 2); - checkContents(contents); - assert.strictEqual(contents[0].Key, 'key1'); - assert.strictEqual(contents[1].Key, 'key1old'); - - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'orphan', prefix }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + + const data = JSON.parse(response.body); + assert.strictEqual(data.IsTruncated, false); + assert(!data.NextMarker); + assert.strictEqual(data.MaxKeys, 1000); + assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); + assert.strictEqual(data.Prefix, prefix); + + const contents = data.Contents; + assert.strictEqual(contents.length, 2); + checkContents(contents); + assert.strictEqual(contents[0].Key, 'key1'); + assert.strictEqual(contents[1].Key, 'key1old'); + + return done(); + }, + ); }); it('should return the orphan delete markers before a defined date', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { - 'list-type': 'orphan', - 'before-date': date, - }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - - const data = JSON.parse(response.body); - assert.strictEqual(data.IsTruncated, false); - assert(!data.NextMarker); - assert.strictEqual(data.MaxKeys, 1000); - assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); - assert.strictEqual(data.Contents.length, 3); - assert.strictEqual(data.BeforeDate, date); - - const contents = data.Contents; - checkContents(contents); - assert.strictEqual(contents[0].Key, 'key0old'); - assert.strictEqual(contents[1].Key, 'key1old'); - assert.strictEqual(contents[2].Key, 'key2old'); - - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { + 'list-type': 'orphan', + 'before-date': date, + }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + + const data = JSON.parse(response.body); + assert.strictEqual(data.IsTruncated, false); + assert(!data.NextMarker); + assert.strictEqual(data.MaxKeys, 1000); + assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); + assert.strictEqual(data.Contents.length, 3); + assert.strictEqual(data.BeforeDate, date); + + const contents = data.Contents; + checkContents(contents); + assert.strictEqual(contents[0].Key, 'key0old'); + assert.strictEqual(contents[1].Key, 'key1old'); + assert.strictEqual(contents[2].Key, 'key2old'); + + return done(); + }, + ); }); it('should truncate list of orphan delete markers before a defined date', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { - 'list-type': 'orphan', - 'before-date': date, - 'max-keys': '1', - }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - - const data = JSON.parse(response.body); - assert.strictEqual(data.IsTruncated, true); - assert.strictEqual(data.NextMarker, 'key0old'); - assert.strictEqual(data.MaxKeys, 1); - assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); - assert.strictEqual(data.BeforeDate, date); - assert.strictEqual(data.Contents.length, 1); - - const contents = data.Contents; - checkContents(contents); - assert.strictEqual(contents[0].Key, 'key0old'); - - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { + 'list-type': 'orphan', + 'before-date': date, + 'max-keys': '1', + }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + + const data = JSON.parse(response.body); + assert.strictEqual(data.IsTruncated, true); + assert.strictEqual(data.NextMarker, 'key0old'); + assert.strictEqual(data.MaxKeys, 1); + assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); + assert.strictEqual(data.BeforeDate, date); + assert.strictEqual(data.Contents.length, 1); + + const contents = data.Contents; + checkContents(contents); + assert.strictEqual(contents[0].Key, 'key0old'); + + return done(); + }, + ); }); it('should return the second truncate list of orphan delete markers before a defined date', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'orphan', 'before-date': date, 'max-keys': '1', 'marker': 'key0old' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - - const data = JSON.parse(response.body); - assert.strictEqual(data.IsTruncated, true); - assert.strictEqual(data.Marker, 'key0old'); - assert.strictEqual(data.NextMarker, 'key1old'); - assert.strictEqual(data.MaxKeys, 1); - assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); - assert.strictEqual(data.Contents.length, 1); - - const contents = data.Contents; - checkContents(contents); - assert.strictEqual(contents[0].Key, 'key1old'); - assert.strictEqual(data.BeforeDate, date); - - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'orphan', 'before-date': date, 'max-keys': '1', marker: 'key0old' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + + const data = JSON.parse(response.body); + assert.strictEqual(data.IsTruncated, true); + assert.strictEqual(data.Marker, 'key0old'); + assert.strictEqual(data.NextMarker, 'key1old'); + assert.strictEqual(data.MaxKeys, 1); + assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); + assert.strictEqual(data.Contents.length, 1); + + const contents = data.Contents; + checkContents(contents); + assert.strictEqual(contents[0].Key, 'key1old'); + assert.strictEqual(data.BeforeDate, date); + + return done(); + }, + ); }); it('should return the third truncate list of orphan delete markers before a defined date', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'orphan', 'before-date': date, 'max-keys': '1', 'marker': 'key1old' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - - const data = JSON.parse(response.body); - assert.strictEqual(data.IsTruncated, true); - assert.strictEqual(data.MaxKeys, 1); - assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); - assert.strictEqual(data.Marker, 'key1old'); - assert.strictEqual(data.BeforeDate, date); - assert.strictEqual(data.NextMarker, 'key2old'); - - const contents = data.Contents; - assert.strictEqual(contents.length, 1); - checkContents(contents); - assert.strictEqual(contents[0].Key, 'key2old'); - - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'orphan', 'before-date': date, 'max-keys': '1', marker: 'key1old' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + + const data = JSON.parse(response.body); + assert.strictEqual(data.IsTruncated, true); + assert.strictEqual(data.MaxKeys, 1); + assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); + assert.strictEqual(data.Marker, 'key1old'); + assert.strictEqual(data.BeforeDate, date); + assert.strictEqual(data.NextMarker, 'key2old'); + + const contents = data.Contents; + assert.strictEqual(contents.length, 1); + checkContents(contents); + assert.strictEqual(contents[0].Key, 'key2old'); + + return done(); + }, + ); }); it('should return the fourth and last truncate list of orphan delete markers before a defined date', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'orphan', 'before-date': date, 'max-keys': '1', 'marker': 'key2old' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - - const data = JSON.parse(response.body); - assert.strictEqual(data.IsTruncated, false); - assert.strictEqual(data.MaxKeys, 1); - assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); - assert.strictEqual(data.Marker, 'key2old'); - assert.strictEqual(data.BeforeDate, date); - - const contents = data.Contents; - assert.strictEqual(contents.length, 0); - - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'orphan', 'before-date': date, 'max-keys': '1', marker: 'key2old' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + + const data = JSON.parse(response.body); + assert.strictEqual(data.IsTruncated, false); + assert.strictEqual(data.MaxKeys, 1); + assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); + assert.strictEqual(data.Marker, 'key2old'); + assert.strictEqual(data.BeforeDate, date); + + const contents = data.Contents; + assert.strictEqual(contents.length, 0); + + return done(); + }, + ); }); }); diff --git a/tests/functional/backbeat/listNullVersion.js b/tests/functional/backbeat/listNullVersion.js index 09f092f7a8..05f2a8841d 100644 --- a/tests/functional/backbeat/listNullVersion.js +++ b/tests/functional/backbeat/listNullVersion.js @@ -31,44 +31,68 @@ async function getCredentials() { describe('listLifecycle if null version', () => { let versionForKey2; - before(done => async.series([ - next => getCredentials().then(creds => { - credentials = creds; - next(); - }).catch(next), - next => s3.send(new CreateBucketCommand({ Bucket: testBucket })) - .then(() => next()) - .catch(next), - next => s3.send(new PutObjectCommand({ Bucket: testBucket, Key: 'key1', Body: '123' })) - .then(() => next()) - .catch(next), - next => s3.send(new PutObjectCommand({ Bucket: testBucket, Key: 'key2', Body: '123' })) - .then(() => next()) - .catch(next), - next => s3.send(new PutBucketVersioningCommand({ - Bucket: testBucket, - VersioningConfiguration: { Status: 'Enabled' }, - })) - .then(() => next()) - .catch(next), - next => s3.send(new PutObjectCommand({ Bucket: testBucket, Key: 'key1', Body: '123' })) - .then(data => - // delete version to create a null current version for key1. - s3.send(new DeleteObjectCommand({ - Bucket: testBucket, - Key: 'key1', - VersionId: data.VersionId - })) - ) - .then(() => next()) - .catch(next), - next => s3.send(new PutObjectCommand({ Bucket: testBucket, Key: 'key2', Body: '123' })) - .then(data => { - versionForKey2 = data.VersionId; - next(); - }) - .catch(next), - ], done)); + before(done => + async.series( + [ + next => + getCredentials() + .then(creds => { + credentials = creds; + next(); + }) + .catch(next), + next => + s3 + .send(new CreateBucketCommand({ Bucket: testBucket })) + .then(() => next()) + .catch(next), + next => + s3 + .send(new PutObjectCommand({ Bucket: testBucket, Key: 'key1', Body: '123' })) + .then(() => next()) + .catch(next), + next => + s3 + .send(new PutObjectCommand({ Bucket: testBucket, Key: 'key2', Body: '123' })) + .then(() => next()) + .catch(next), + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: testBucket, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ) + .then(() => next()) + .catch(next), + next => + s3 + .send(new PutObjectCommand({ Bucket: testBucket, Key: 'key1', Body: '123' })) + .then(data => + // delete version to create a null current version for key1. + s3.send( + new DeleteObjectCommand({ + Bucket: testBucket, + Key: 'key1', + VersionId: data.VersionId, + }), + ), + ) + .then(() => next()) + .catch(next), + next => + s3 + .send(new PutObjectCommand({ Bucket: testBucket, Key: 'key2', Body: '123' })) + .then(data => { + versionForKey2 = data.VersionId; + next(); + }) + .catch(next), + ], + done, + ), + ); after(async () => { await removeAllVersionsPromise({ Bucket: testBucket }); @@ -76,55 +100,61 @@ describe('listLifecycle if null version', () => { }); it('should return the null noncurrent versions', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'noncurrent' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); - - assert.strictEqual(data.IsTruncated, false); - assert(!data.NextKeyMarker); - assert.strictEqual(data.MaxKeys, 1000); - - const contents = data.Contents; - assert.strictEqual(contents.length, 1); - assert.strictEqual(contents[0].Key, 'key2'); - assert.strictEqual(contents[0].VersionId, 'null'); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'noncurrent' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); + + assert.strictEqual(data.IsTruncated, false); + assert(!data.NextKeyMarker); + assert.strictEqual(data.MaxKeys, 1000); + + const contents = data.Contents; + assert.strictEqual(contents.length, 1); + assert.strictEqual(contents[0].Key, 'key2'); + assert.strictEqual(contents[0].VersionId, 'null'); + return done(); + }, + ); }); it('should return the null current versions', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'current' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); - - assert.strictEqual(data.IsTruncated, false); - assert(!data.NextKeyMarker); - assert.strictEqual(data.MaxKeys, 1000); - - const contents = data.Contents; - assert.strictEqual(contents.length, 2); - - const firstKey = contents[0]; - assert.strictEqual(firstKey.Key, 'key1'); - assert.strictEqual(firstKey.VersionId, 'null'); - - const secondKey = contents[1]; - assert.strictEqual(secondKey.Key, 'key2'); - assert.strictEqual(secondKey.VersionId, versionForKey2); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'current' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); + + assert.strictEqual(data.IsTruncated, false); + assert(!data.NextKeyMarker); + assert.strictEqual(data.MaxKeys, 1000); + + const contents = data.Contents; + assert.strictEqual(contents.length, 2); + + const firstKey = contents[0]; + assert.strictEqual(firstKey.Key, 'key1'); + assert.strictEqual(firstKey.VersionId, 'null'); + + const secondKey = contents[1]; + assert.strictEqual(secondKey.Key, 'key2'); + assert.strictEqual(secondKey.VersionId, versionForKey2); + return done(); + }, + ); }); }); @@ -133,32 +163,51 @@ describe('listLifecycle with null current version after versioning suspended', ( const nullObjectBucket = 'bucket-for-list-lifecycle-current-null-tests'; const keyName = 'key0'; - before(done => async.series([ - next => s3.send(new CreateBucketCommand({ Bucket: nullObjectBucket })) - .then(() => next()) - .catch(next), - next => s3.send(new PutBucketVersioningCommand({ - Bucket: nullObjectBucket, - VersioningConfiguration: { Status: 'Enabled' }, - })) - .then(() => next()) - .catch(next), - next => s3.send(new PutObjectCommand({ Bucket: nullObjectBucket, Key: keyName })) - .then(data => { - expectedVersionId = data.VersionId; - next(); - }) - .catch(next), - next => s3.send(new PutBucketVersioningCommand({ - Bucket: nullObjectBucket, - VersioningConfiguration: { Status: 'Suspended' }, - })) - .then(() => next()) - .catch(next), - next => s3.send(new PutObjectCommand({ Bucket: nullObjectBucket, Key: keyName })) - .then(() => next()) - .catch(next), - ], done)); + before(done => + async.series( + [ + next => + s3 + .send(new CreateBucketCommand({ Bucket: nullObjectBucket })) + .then(() => next()) + .catch(next), + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: nullObjectBucket, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ) + .then(() => next()) + .catch(next), + next => + s3 + .send(new PutObjectCommand({ Bucket: nullObjectBucket, Key: keyName })) + .then(data => { + expectedVersionId = data.VersionId; + next(); + }) + .catch(next), + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: nullObjectBucket, + VersioningConfiguration: { Status: 'Suspended' }, + }), + ) + .then(() => next()) + .catch(next), + next => + s3 + .send(new PutObjectCommand({ Bucket: nullObjectBucket, Key: keyName })) + .then(() => next()) + .catch(next), + ], + done, + ), + ); after(async () => { await removeAllVersionsPromise({ Bucket: nullObjectBucket }); @@ -166,46 +215,52 @@ describe('listLifecycle with null current version after versioning suspended', ( }); it('should return list of current versions when bucket has a null current version', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: nullObjectBucket, - queryObj: { 'list-type': 'current' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); - - assert.strictEqual(data.IsTruncated, false); - assert(!data.NextKeyMarker); - assert.strictEqual(data.MaxKeys, 1000); - assert.strictEqual(data.Contents.length, 1); - const key = data.Contents[0]; - assert.strictEqual(key.Key, keyName); - assert.strictEqual(key.VersionId, 'null'); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: nullObjectBucket, + queryObj: { 'list-type': 'current' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); + + assert.strictEqual(data.IsTruncated, false); + assert(!data.NextKeyMarker); + assert.strictEqual(data.MaxKeys, 1000); + assert.strictEqual(data.Contents.length, 1); + const key = data.Contents[0]; + assert.strictEqual(key.Key, keyName); + assert.strictEqual(key.VersionId, 'null'); + return done(); + }, + ); }); it('should return list of non-current versions when bucket has a null current version', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: nullObjectBucket, - queryObj: { 'list-type': 'noncurrent' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); - - assert.strictEqual(data.IsTruncated, false); - assert(!data.NextKeyMarker); - assert.strictEqual(data.MaxKeys, 1000); - assert.strictEqual(data.Contents.length, 1); - const key = data.Contents[0]; - assert.strictEqual(key.Key, keyName); - assert.strictEqual(key.VersionId, expectedVersionId); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: nullObjectBucket, + queryObj: { 'list-type': 'noncurrent' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); + + assert.strictEqual(data.IsTruncated, false); + assert(!data.NextKeyMarker); + assert.strictEqual(data.MaxKeys, 1000); + assert.strictEqual(data.Contents.length, 1); + const key = data.Contents[0]; + assert.strictEqual(key.Key, keyName); + assert.strictEqual(key.VersionId, expectedVersionId); + return done(); + }, + ); }); }); diff --git a/tests/functional/backbeat/utils.js b/tests/functional/backbeat/utils.js index 60ef8aa113..3bf40c2f37 100644 --- a/tests/functional/backbeat/utils.js +++ b/tests/functional/backbeat/utils.js @@ -1,7 +1,9 @@ const { makeRequest } = require('../raw-node/utils/makeRequest'); const ipAddress = process.env.IP ? process.env.IP : '127.0.0.1'; -const { models: { ObjectMD } } = require('arsenal'); +const { + models: { ObjectMD }, +} = require('arsenal'); // NOTE: The routes "getMetadata" and "putMetadata" are utilized for modifying the metadata of an object. // This approach is preferred over directly updating the metadata in MongoDB, diff --git a/tests/functional/healthchecks/package.json b/tests/functional/healthchecks/package.json index 6632cf8ef0..034827c5b6 100644 --- a/tests/functional/healthchecks/package.json +++ b/tests/functional/healthchecks/package.json @@ -1,17 +1,16 @@ { - "name": "Test-healthcheck", - "version": "0.1.0", - "description": "Test-healthcheck", - "main": "tests.js", - "private": true, - "repository": "", - "keywords": [ - "test" - ], - "scripts": { - "test": "mocha -t 40000 test/ --exit", - "test-debug": "_mocha -t 40000 test/ --exit" - }, - "author": "" + "name": "Test-healthcheck", + "version": "0.1.0", + "description": "Test-healthcheck", + "main": "tests.js", + "private": true, + "repository": "", + "keywords": [ + "test" + ], + "scripts": { + "test": "mocha -t 40000 test/ --exit", + "test-debug": "_mocha -t 40000 test/ --exit" + }, + "author": "" } - diff --git a/tests/functional/healthchecks/test/checkRoutes.js b/tests/functional/healthchecks/test/checkRoutes.js index 4df9da81b2..0ce8ba92ab 100644 --- a/tests/functional/healthchecks/test/checkRoutes.js +++ b/tests/functional/healthchecks/test/checkRoutes.js @@ -151,8 +151,7 @@ describe('Healthcheck stats', () => { const totalReqs = 5; beforeEach(done => { redis.flushdb(() => { - async.timesSeries(totalReqs, - (n, next) => makeDummyS3Request(next), done); + async.timesSeries(totalReqs, (n, next) => makeDummyS3Request(next), done); }); }); @@ -165,11 +164,9 @@ describe('Healthcheck stats', () => { if (err) { return done(err); } - const expectedStatsRes = { 'requests': totalReqs, '500s': 0, - 'sampleDuration': 30 }; + const expectedStatsRes = { requests: totalReqs, '500s': 0, sampleDuration: 30 }; assert.deepStrictEqual(JSON.parse(res), expectedStatsRes); return done(); }); - }, 500) - ); + }, 500)); }); diff --git a/tests/functional/kmip/serverside_encryption.js b/tests/functional/kmip/serverside_encryption.js index 88badb4733..855b356f06 100644 --- a/tests/functional/kmip/serverside_encryption.js +++ b/tests/functional/kmip/serverside_encryption.js @@ -1,9 +1,4 @@ -const { - S3Client, - PutObjectCommand, - CopyObjectCommand, - CreateMultipartUploadCommand -} = require('@aws-sdk/client-s3'); +const { S3Client, PutObjectCommand, CopyObjectCommand, CreateMultipartUploadCommand } = require('@aws-sdk/client-s3'); const { v4: uuidv4 } = require('uuid'); const config = require('../config.json'); const { auth } = require('arsenal'); @@ -13,7 +8,6 @@ const assert = require('assert'); const logger = { info: msg => process.stdout.write(`${msg}\n`) }; const async = require('async'); - function _createBucket(name, encrypt, done) { const { transport, ipAddress, accessKey, secretKey } = config; const verbose = false; @@ -69,10 +63,8 @@ function _createBucket(name, encrypt, done) { function _buildS3() { const { transport, ipAddress, accessKey, secretKey } = config; - const agent = transport === 'https' - ? new https.Agent({ keepAlive: false }) - : new http.Agent({ keepAlive: false }); - + const agent = transport === 'https' ? new https.Agent({ keepAlive: false }) : new http.Agent({ keepAlive: false }); + return new S3Client({ endpoint: `${transport}://${ipAddress}:8000`, region: 'us-east-1', @@ -107,8 +99,7 @@ function _putObject(bucketName, objectName, encrypt, cb) { .catch(err => cb(err)); } -function _copyObject(sourceBucket, sourceObject, targetBucket, targetObject, - encrypt, cb) { +function _copyObject(sourceBucket, sourceObject, targetBucket, targetObject, encrypt, cb) { const params = { Bucket: targetBucket, CopySource: `/${sourceBucket}/${sourceObject}`, @@ -154,58 +145,62 @@ describe('KMIP backed server-side encryption', () => { it('should create an encrypted bucket', done => { _createBucket(bucketName, true, err => { - assert.equal(err, null, 'Expected success, ' + - `got error ${JSON.stringify(err)}`); + assert.equal(err, null, 'Expected success, ' + `got error ${JSON.stringify(err)}`); done(); }); }); it('should create an encrypted bucket and upload an object', done => { - async.waterfall([ - next => _createBucket(bucketName, true, err => next(err)), - next => _putObject(bucketName, objectName, false, err => next(err)), - ], err => { - assert.equal(err, null, 'Expected success, ' + - `got error ${JSON.stringify(err)}`); - done(); - }); + async.waterfall( + [ + next => _createBucket(bucketName, true, err => next(err)), + next => _putObject(bucketName, objectName, false, err => next(err)), + ], + err => { + assert.equal(err, null, 'Expected success, ' + `got error ${JSON.stringify(err)}`); + done(); + }, + ); }); it('should allow object PUT with SSE header in encrypted bucket', done => { - async.waterfall([ - next => _createBucket(bucketName, true, err => next(err)), - next => _putObject(bucketName, objectName, true, err => next(err)), - ], err => { - assert.equal(err, null, 'Expected success, ' + - `got error ${JSON.stringify(err)}`); - done(); - }); + async.waterfall( + [ + next => _createBucket(bucketName, true, err => next(err)), + next => _putObject(bucketName, objectName, true, err => next(err)), + ], + err => { + assert.equal(err, null, 'Expected success, ' + `got error ${JSON.stringify(err)}`); + done(); + }, + ); }); it('should allow object copy with SSE header in encrypted bucket', done => { - async.waterfall([ - next => _createBucket(bucketName, false, err => next(err)), - next => _putObject(bucketName, objectName, false, err => next(err)), - next => _createBucket(`${bucketName}2`, true, err => next(err)), - next => _copyObject(bucketName, objectName, `${bucketName}2`, - `${objectName}2`, true, err => next(err)), - ], err => { - assert.equal(err, null, 'Expected success, ' + - `got error ${JSON.stringify(err)}`); - done(); - }); + async.waterfall( + [ + next => _createBucket(bucketName, false, err => next(err)), + next => _putObject(bucketName, objectName, false, err => next(err)), + next => _createBucket(`${bucketName}2`, true, err => next(err)), + next => _copyObject(bucketName, objectName, `${bucketName}2`, `${objectName}2`, true, err => next(err)), + ], + err => { + assert.equal(err, null, 'Expected success, ' + `got error ${JSON.stringify(err)}`); + done(); + }, + ); }); - it('should allow creating mpu with SSE header ' + - 'in encrypted bucket', done => { - async.waterfall([ - next => _createBucket(bucketName, true, err => next(err)), - next => _initiateMultipartUpload(bucketName, objectName, - true, err => next(err)), - ], err => { - assert.equal(err, null, 'Expected success, ' + - `got error ${JSON.stringify(err)}`); - done(); - }); + it('should allow creating mpu with SSE header ' + 'in encrypted bucket', done => { + async.waterfall( + [ + next => _createBucket(bucketName, true, err => next(err)), + next => _initiateMultipartUpload(bucketName, objectName, true, err => next(err)), + ], + err => { + assert.equal(err, null, 'Expected success, ' + `got error ${JSON.stringify(err)}`); + done(); + }, + ); }); }); diff --git a/tests/functional/metadata/MixedVersionFormat.js b/tests/functional/metadata/MixedVersionFormat.js index a72c708375..b5bf665f5f 100644 --- a/tests/functional/metadata/MixedVersionFormat.js +++ b/tests/functional/metadata/MixedVersionFormat.js @@ -1,11 +1,11 @@ const assert = require('assert'); const async = require('async'); -const { - PutObjectCommand, - GetObjectCommand, - ListObjectsCommand, +const { + PutObjectCommand, + GetObjectCommand, + ListObjectsCommand, PutBucketVersioningCommand, - ListObjectVersionsCommand + ListObjectVersionsCommand, } = require('@aws-sdk/client-s3'); const withV4 = require('../aws-node-sdk/test/support/withV4'); const BucketUtility = require('../aws-node-sdk/lib/utility/bucket-util'); @@ -16,8 +16,8 @@ const replicaSetHosts = 'localhost:27017,localhost:27018,localhost:27019'; const writeConcern = 'majority'; const replicaSet = 'rs0'; const readPreference = 'primary'; -const mongoUrl = `mongodb://${replicaSetHosts}/?w=${writeConcern}&` + - `replicaSet=${replicaSet}&readPreference=${readPreference}`; +const mongoUrl = + `mongodb://${replicaSetHosts}/?w=${writeConcern}&` + `replicaSet=${replicaSet}&readPreference=${readPreference}`; /** * These tests are intended to see if the vFormat of buckets is respected @@ -47,61 +47,74 @@ describe('Mongo backend mixed bucket format versions', () => { function updateBucketVFormat(bucketName, vFormat) { const db = mongoClient.db('metadata'); - return db.collection('__metastore') - .updateOne({ + return db.collection('__metastore').updateOne( + { _id: bucketName, - }, { + }, + { $set: { vFormat }, - }, {}); + }, + {}, + ); } function getObject(bucketName, key, cb) { const db = mongoClient.db('metadata'); - return db.collection(bucketName) - .findOne({ - _id: key, - }, {}).then(doc => { - if (!doc) { - return cb(errors.NoSuchKey); - } - return cb(null, doc.value); - }).catch(err => cb(err)); + return db + .collection(bucketName) + .findOne( + { + _id: key, + }, + {}, + ) + .then(doc => { + if (!doc) { + return cb(errors.NoSuchKey); + } + return cb(null, doc.value); + }) + .catch(err => cb(err)); } before(done => { - MongoClient.connect(mongoUrl, {}).then(client => { - mongoClient = client; - bucketUtil = new BucketUtility('default', sigCfg); - s3 = bucketUtil.s3; - return done(); - }).catch(err => done(err)); + MongoClient.connect(mongoUrl, {}) + .then(client => { + mongoClient = client; + bucketUtil = new BucketUtility('default', sigCfg); + s3 = bucketUtil.s3; + return done(); + }) + .catch(err => done(err)); }); beforeEach(() => { process.stdout.write('Creating buckets'); - return bucketUtil.createMany(['v0-bucket', 'v1-bucket']) - .then(async () => { - process.stdout.write('Updating bucket vFormat'); - await updateBucketVFormat('v0-bucket', 'v0'); - await updateBucketVFormat('v1-bucket', 'v1'); - }) - .catch(err => { - process.stdout.write('Error in afterEach'); - throw err; - }); + return bucketUtil + .createMany(['v0-bucket', 'v1-bucket']) + .then(async () => { + process.stdout.write('Updating bucket vFormat'); + await updateBucketVFormat('v0-bucket', 'v0'); + await updateBucketVFormat('v1-bucket', 'v1'); + }) + .catch(err => { + process.stdout.write('Error in afterEach'); + throw err; + }); }); afterEach(() => { process.stdout.write('Emptying buckets'); - return bucketUtil.emptyMany(['v0-bucket', 'v1-bucket']) - .then(() => { - process.stdout.write('Deleting buckets'); - return bucketUtil.deleteMany(['v0-bucket', 'v1-bucket']); - }) - .catch(err => { - process.stdout.write('Error in afterEach'); - throw err; - }); + return bucketUtil + .emptyMany(['v0-bucket', 'v1-bucket']) + .then(() => { + process.stdout.write('Deleting buckets'); + return bucketUtil.deleteMany(['v0-bucket', 'v1-bucket']); + }) + .catch(err => { + process.stdout.write('Error in afterEach'); + throw err; + }); }); after(async () => { @@ -112,129 +125,137 @@ describe('Mongo backend mixed bucket format versions', () => { it(`Should perform operations on non versioned bucket in ${vFormat} format`, done => { const paramsObj1 = { Bucket: `${vFormat}-bucket`, - Key: `${vFormat}-object-1` + Key: `${vFormat}-object-1`, }; const paramsObj2 = { Bucket: `${vFormat}-bucket`, - Key: `${vFormat}-object-2` + Key: `${vFormat}-object-2`, }; const masterKey = vFormat === 'v0' ? `${vFormat}-object-1` : `\x7fM${vFormat}-object-1`; - async.series([ - next => { - s3.send(new PutObjectCommand(paramsObj1)) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new PutObjectCommand(paramsObj2)) - .then(() => next()) - .catch(next); - }, - // check if data stored in the correct format - next => getObject(`${vFormat}-bucket`, masterKey, (err, doc) => { - assert.ifError(err); - assert.strictEqual(doc.key, `${vFormat}-object-1`); - return next(); - }), - // test if we can get object - next => { - s3.send(new GetObjectCommand(paramsObj1)) - .then(() => next()) - .catch(next); - }, - // test if we can list objects - next => { - s3.send(new ListObjectsCommand({ Bucket: `${vFormat}-bucket` })) - .then(data => { - assert.strictEqual(data.Contents.length, 2); - const keys = data.Contents.map(obj => obj.Key); - assert(keys.includes(`${vFormat}-object-1`)); - assert(keys.includes(`${vFormat}-object-2`)); - next(); - }) - .catch(next); - } - ], done); + async.series( + [ + next => { + s3.send(new PutObjectCommand(paramsObj1)) + .then(() => next()) + .catch(next); + }, + next => { + s3.send(new PutObjectCommand(paramsObj2)) + .then(() => next()) + .catch(next); + }, + // check if data stored in the correct format + next => + getObject(`${vFormat}-bucket`, masterKey, (err, doc) => { + assert.ifError(err); + assert.strictEqual(doc.key, `${vFormat}-object-1`); + return next(); + }), + // test if we can get object + next => { + s3.send(new GetObjectCommand(paramsObj1)) + .then(() => next()) + .catch(next); + }, + // test if we can list objects + next => { + s3.send(new ListObjectsCommand({ Bucket: `${vFormat}-bucket` })) + .then(data => { + assert.strictEqual(data.Contents.length, 2); + const keys = data.Contents.map(obj => obj.Key); + assert(keys.includes(`${vFormat}-object-1`)); + assert(keys.includes(`${vFormat}-object-2`)); + next(); + }) + .catch(next); + }, + ], + done, + ); }); it(`Should perform operations on versioned bucket in ${vFormat} format`, done => { const paramsObj1 = { Bucket: `${vFormat}-bucket`, - Key: `${vFormat}-object-1` + Key: `${vFormat}-object-1`, }; const paramsObj2 = { Bucket: `${vFormat}-bucket`, - Key: `${vFormat}-object-2` + Key: `${vFormat}-object-2`, }; const versioningParams = { Bucket: `${vFormat}-bucket`, VersioningConfiguration: { - Status: 'Enabled', - } + Status: 'Enabled', + }, }; const masterKey = vFormat === 'v0' ? `${vFormat}-object-1` : `\x7fM${vFormat}-object-1`; - - async.series([ - next => { - s3.send(new PutBucketVersioningCommand(versioningParams)) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new PutObjectCommand(paramsObj1)) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new PutObjectCommand(paramsObj1)) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new PutObjectCommand(paramsObj2)) - .then(() => next()) - .catch(next); - }, - // check if data stored in the correct version format - next => getObject(`${vFormat}-bucket`, masterKey, (err, doc) => { - assert.ifError(err); - assert.strictEqual(doc.key, `${vFormat}-object-1`); - return next(); - }), - // test if we can get object - next => { - s3.send(new GetObjectCommand(paramsObj1)) - .then(() => next()) - .catch(next); - }, - // test if we can list objects - next => { - s3.send(new ListObjectsCommand({ Bucket: `${vFormat}-bucket` })) - .then(data => { - assert.strictEqual(data.Contents.length, 2); - const keys = data.Contents.map(obj => obj.Key); - assert(keys.includes(`${vFormat}-object-1`)); - assert(keys.includes(`${vFormat}-object-2`)); - next(); - }) - .catch(next); - }, - // test if we can list object versions - next => { - s3.send(new ListObjectVersionsCommand({ Bucket: `${vFormat}-bucket` })) - .then(data => { - assert.strictEqual(data.Versions.length, 3); - const versionPerObject = {}; - data.Versions.forEach(version => { - versionPerObject[version.Key] = (versionPerObject[version.Key] || 0) + 1; - }); - assert.strictEqual(versionPerObject[`${vFormat}-object-1`], 2); - assert.strictEqual(versionPerObject[`${vFormat}-object-2`], 1); - next(); - }) - .catch(next); - } - ], done); + + async.series( + [ + next => { + s3.send(new PutBucketVersioningCommand(versioningParams)) + .then(() => next()) + .catch(next); + }, + next => { + s3.send(new PutObjectCommand(paramsObj1)) + .then(() => next()) + .catch(next); + }, + next => { + s3.send(new PutObjectCommand(paramsObj1)) + .then(() => next()) + .catch(next); + }, + next => { + s3.send(new PutObjectCommand(paramsObj2)) + .then(() => next()) + .catch(next); + }, + // check if data stored in the correct version format + next => + getObject(`${vFormat}-bucket`, masterKey, (err, doc) => { + assert.ifError(err); + assert.strictEqual(doc.key, `${vFormat}-object-1`); + return next(); + }), + // test if we can get object + next => { + s3.send(new GetObjectCommand(paramsObj1)) + .then(() => next()) + .catch(next); + }, + // test if we can list objects + next => { + s3.send(new ListObjectsCommand({ Bucket: `${vFormat}-bucket` })) + .then(data => { + assert.strictEqual(data.Contents.length, 2); + const keys = data.Contents.map(obj => obj.Key); + assert(keys.includes(`${vFormat}-object-1`)); + assert(keys.includes(`${vFormat}-object-2`)); + next(); + }) + .catch(next); + }, + // test if we can list object versions + next => { + s3.send(new ListObjectVersionsCommand({ Bucket: `${vFormat}-bucket` })) + .then(data => { + assert.strictEqual(data.Versions.length, 3); + const versionPerObject = {}; + data.Versions.forEach(version => { + versionPerObject[version.Key] = (versionPerObject[version.Key] || 0) + 1; + }); + assert.strictEqual(versionPerObject[`${vFormat}-object-1`], 2); + assert.strictEqual(versionPerObject[`${vFormat}-object-2`], 1); + next(); + }) + .catch(next); + }, + ], + done, + ); }); }); }); diff --git a/tests/functional/metadata/MongoClientInterface.js b/tests/functional/metadata/MongoClientInterface.js index a8ce1456a2..b883c14986 100644 --- a/tests/functional/metadata/MongoClientInterface.js +++ b/tests/functional/metadata/MongoClientInterface.js @@ -2,9 +2,7 @@ const assert = require('assert'); const async = require('async'); const MongoClient = require('mongodb').MongoClient; -const { - MongoClientInterface, -} = require('arsenal').storage.metadata.mongoclient; +const { MongoClientInterface } = require('arsenal').storage.metadata.mongoclient; const { errors } = require('arsenal'); const log = require('./utils/fakeLogger'); @@ -13,8 +11,8 @@ const replicaSetHosts = 'localhost:27017,localhost:27018,localhost:27019'; const writeConcern = 'majority'; const replicaSet = 'rs0'; const readPreference = 'primary'; -const mongoUrl = `mongodb://${replicaSetHosts}/?w=${writeConcern}&` + - `replicaSet=${replicaSet}&readPreference=${readPreference}`; +const mongoUrl = + `mongodb://${replicaSetHosts}/?w=${writeConcern}&` + `replicaSet=${replicaSet}&readPreference=${readPreference}`; const VID_SEP = '\0'; const TEST_DB = 'test'; @@ -41,13 +39,14 @@ const objVal = { const updatedObjVal = { updated: true }; -const runIfMongo = - process.env.S3METADATA === 'mongodb' ? describe : describe.skip; +const runIfMongo = process.env.S3METADATA === 'mongodb' ? describe : describe.skip; function unescapeJSON(obj) { - return JSON.parse(JSON.stringify(obj). - replace(/\uFF04/g, '$'). - replace(/\uFF0E/g, '.')); + return JSON.parse( + JSON.stringify(obj) + .replace(/\uFF04/g, '$') + .replace(/\uFF0E/g, '.'), + ); } function getCaseMethod(number) { @@ -72,25 +71,28 @@ runIfMongo('MongoClientInterface', () => { if (params && params.versionId) { objName = `${objName}${VID_SEP}${params.versionId}`; } - collection.findOne({ - _id: objName, - }, {}).then(doc => { - if (!doc) { - return cb(errors.NoSuchKey); - } - if (doc.value.tags) { - // eslint-disable-next-line - doc.value.tags = unescapeJSON(doc.value.tags); - } - return cb(null, doc.value); - }).catch(err => cb(err)); + collection + .findOne( + { + _id: objName, + }, + {}, + ) + .then(doc => { + if (!doc) { + return cb(errors.NoSuchKey); + } + if (doc.value.tags) { + // eslint-disable-next-line + doc.value.tags = unescapeJSON(doc.value.tags); + } + return cb(null, doc.value); + }) + .catch(err => cb(err)); } function checkVersionAndMasterMatch(versionId, cb) { - async.parallel([ - next => getObject({}, next), - next => getObject({ versionId }, next), - ], (err, res) => { + async.parallel([next => getObject({}, next), next => getObject({ versionId }, next)], (err, res) => { if (err) { return cb(err); } @@ -104,41 +106,42 @@ runIfMongo('MongoClientInterface', () => { function putObject(caseNum, params, objectValue, cb) { const method = getCaseMethod(caseNum); const dupeObjVal = Object.assign({}, objectValue || objVal); - mongoClientInterface[method](collection, BUCKET_NAME, OBJECT_NAME, - dupeObjVal, params, log, (err, res) => { - if (err) { - return cb(err); - } - let parsedRes; - if (res) { - try { - parsedRes = JSON.parse(res); - } catch (error) { - return cb(error); - } + mongoClientInterface[method](collection, BUCKET_NAME, OBJECT_NAME, dupeObjVal, params, log, (err, res) => { + if (err) { + return cb(err); + } + let parsedRes; + if (res) { + try { + parsedRes = JSON.parse(res); + } catch (error) { + return cb(error); } - return cb(null, parsedRes); - }); + } + return cb(null, parsedRes); + }); } function checkNewPutObject(caseNum, params, cb) { const method = getCaseMethod(caseNum); const bucket = 'a'; const key = 'b'; - async.series([ - next => mongoClientInterface[method]( - collection, bucket, key, updatedObjVal, params, log, next), - next => { - collection.findOne({ _id: key }, (err, result) => { - if (err) { - return next(err); - } - assert.strictEqual(result._id, key); - assert(result.value.updated); - return next(); - }); - }, - ], cb); + async.series( + [ + next => mongoClientInterface[method](collection, bucket, key, updatedObjVal, params, log, next), + next => { + collection.findOne({ _id: key }, (err, result) => { + if (err) { + return next(err); + } + assert.strictEqual(result._id, key); + assert(result.value.updated); + return next(); + }); + }, + ], + cb, + ); } before(done => { @@ -180,11 +183,13 @@ runIfMongo('MongoClientInterface', () => { describe('::putObjectVerCase1', () => { it('should put new metadata and update master', done => { - async.waterfall([ - next => putObject(1, {}, null, - (err, res) => next(err, res.versionId)), - (id, next) => checkVersionAndMasterMatch(id, next), - ], done); + async.waterfall( + [ + next => putObject(1, {}, null, (err, res) => next(err, res.versionId)), + (id, next) => checkVersionAndMasterMatch(id, next), + ], + done, + ); }); }); @@ -192,130 +197,152 @@ runIfMongo('MongoClientInterface', () => { it('should put new metadata', done => checkNewPutObject(2, {}, done)); it('should set new version id for master', done => { - async.waterfall([ - // first create new ver and master - next => putObject(1, {}, null, next), - // check master and version were created and match - (res, next) => checkVersionAndMasterMatch(res.versionId, - err => next(err, res.versionId)), - // call ver case 2 - (id, next) => putObject(2, {}, null, (err, res) => { - if (err) { - return next(err); - } - assert(id !== res.versionId); - return next(null, res.versionId); - }), - // assert master updated with new version id - (newId, next) => getObject({}, (err, res) => { - if (err) { - return next(err); - } - assert.strictEqual(res.versionId, newId); - return next(null, newId); - }), - // new version entry should not have been created - (id, next) => getObject({ versionId: id }, err => { - assert(err); - assert(err.is.NoSuchKey); - return next(); - }), - ], done); + async.waterfall( + [ + // first create new ver and master + next => putObject(1, {}, null, next), + // check master and version were created and match + (res, next) => checkVersionAndMasterMatch(res.versionId, err => next(err, res.versionId)), + // call ver case 2 + (id, next) => + putObject(2, {}, null, (err, res) => { + if (err) { + return next(err); + } + assert(id !== res.versionId); + return next(null, res.versionId); + }), + // assert master updated with new version id + (newId, next) => + getObject({}, (err, res) => { + if (err) { + return next(err); + } + assert.strictEqual(res.versionId, newId); + return next(null, newId); + }), + // new version entry should not have been created + (id, next) => + getObject({ versionId: id }, err => { + assert(err); + assert(err.is.NoSuchKey); + return next(); + }), + ], + done, + ); }); }); describe('::putObjectVerCase3', () => { - it('should put new metadata', done => - checkNewPutObject(3, { versionId: VERSION_ID }, done)); + it('should put new metadata', done => checkNewPutObject(3, { versionId: VERSION_ID }, done)); it('should put new metadata and not update master', done => { - async.waterfall([ - // first create new ver and master - next => putObject(1, {}, null, next), - // check master and version were created and match - (res, next) => checkVersionAndMasterMatch(res.versionId, - err => next(err, res.versionId)), - // call ver case 3 - (id, next) => putObject(3, { versionId: VERSION_ID }, null, - (err, res) => { - if (err) { - return next(err); - } - // assert new version id created - assert(id !== res.versionId); - assert.strictEqual(res.versionId, VERSION_ID); - return next(null, id); - }), - // assert master did not update and matches old initial version - (oldId, next) => getObject({}, (err, res) => { - if (err) { - return next(err); - } - assert.strictEqual(oldId, res.versionId); - return next(); - }), - // assert new version was created - next => getObject({ versionId: VERSION_ID }, (err, res) => { - if (err) { - return next(err); - } - assert(res); - assert.strictEqual(res.versionId, VERSION_ID); - return next(); - }), - ], done); + async.waterfall( + [ + // first create new ver and master + next => putObject(1, {}, null, next), + // check master and version were created and match + (res, next) => checkVersionAndMasterMatch(res.versionId, err => next(err, res.versionId)), + // call ver case 3 + (id, next) => + putObject(3, { versionId: VERSION_ID }, null, (err, res) => { + if (err) { + return next(err); + } + // assert new version id created + assert(id !== res.versionId); + assert.strictEqual(res.versionId, VERSION_ID); + return next(null, id); + }), + // assert master did not update and matches old initial version + (oldId, next) => + getObject({}, (err, res) => { + if (err) { + return next(err); + } + assert.strictEqual(oldId, res.versionId); + return next(); + }), + // assert new version was created + next => + getObject({ versionId: VERSION_ID }, (err, res) => { + if (err) { + return next(err); + } + assert(res); + assert.strictEqual(res.versionId, VERSION_ID); + return next(); + }), + ], + done, + ); }); - it('should put new metadata and update master if version id matches', - done => { - async.waterfall([ - // first create new ver and master - next => putObject(1, {}, null, next), - // check master and version were created and match - (res, next) => checkVersionAndMasterMatch(res.versionId, - err => next(err, res.versionId)), - // call ver case 3 w/ same version id and update - (id, next) => mongoClientInterface.putObjectVerCase3(collection, - BUCKET_NAME, OBJECT_NAME, updatedObjVal, - { versionId: id }, log, err => next(err, id)), - (oldId, next) => getObject({}, (err, res) => { - if (err) { - return next(err); - } - // assert updated - assert(res); - assert(res.updated); - // assert same version id - assert.strictEqual(oldId, res.versionId); - return next(); - }), - ], done); + it('should put new metadata and update master if version id matches', done => { + async.waterfall( + [ + // first create new ver and master + next => putObject(1, {}, null, next), + // check master and version were created and match + (res, next) => checkVersionAndMasterMatch(res.versionId, err => next(err, res.versionId)), + // call ver case 3 w/ same version id and update + (id, next) => + mongoClientInterface.putObjectVerCase3( + collection, + BUCKET_NAME, + OBJECT_NAME, + updatedObjVal, + { versionId: id }, + log, + err => next(err, id), + ), + (oldId, next) => + getObject({}, (err, res) => { + if (err) { + return next(err); + } + // assert updated + assert(res); + assert(res.updated); + // assert same version id + assert.strictEqual(oldId, res.versionId); + return next(); + }), + ], + done, + ); }); }); describe('::putObjectVerCase4', () => { function putAndCheckCase4(versionId, cb) { const objectValue = Object.assign({}, objVal, { versionId }); - async.waterfall([ - // put object - next => putObject(4, { versionId }, objectValue, (err, res) => { - if (err) { - return next(err); - } - return next(null, res.versionId); - }), - (id, next) => getObject({}, (err, res) => { - if (err) { - return next(err); - } - // assert PHD was placed on master - assert(res); - assert.strictEqual(res.isPHD, true); - // assert same version id as master - assert.strictEqual(id, res.versionId); - return next(); - }), - ], cb); + async.waterfall( + [ + // put object + next => + putObject(4, { versionId }, objectValue, (err, res) => { + if (err) { + return next(err); + } + return next(null, res.versionId); + }), + (id, next) => + getObject({}, (err, res) => { + if (err) { + return next(err); + } + // assert PHD was placed on master + assert(res); + assert.strictEqual(res.isPHD, true); + // assert same version id as master + assert.strictEqual(id, res.versionId); + return next(); + }), + ], + cb, + ); } it('should put new metadata and update master', done => { @@ -330,11 +357,14 @@ runIfMongo('MongoClientInterface', () => { const suffix = `22019.${count++}`; return `${prefix}${repID}${suffix}`; } - async.series([ - next => putAndCheckCase4(getNewVersion(), next), - next => putAndCheckCase4(getNewVersion(), next), - next => putAndCheckCase4(getNewVersion(), next), - ], done); + async.series( + [ + next => putAndCheckCase4(getNewVersion(), next), + next => putAndCheckCase4(getNewVersion(), next), + next => putAndCheckCase4(getNewVersion(), next), + ], + done, + ); }); }); }); diff --git a/tests/functional/raw-node/package.json b/tests/functional/raw-node/package.json index 1c20bf3262..4640f5ac23 100644 --- a/tests/functional/raw-node/package.json +++ b/tests/functional/raw-node/package.json @@ -1,23 +1,23 @@ { - "name": "Test-rawnode", - "version": "0.1.0", - "description": "Test-rawnode", - "main": "tests.js", - "private": true, - "repository": "", - "keywords": [ - "test" - ], - "scripts": { - "test-aws": "AWS_ON_AIR=true mocha -t 40000 test/ --exit", - "test-gcp": "mocha -t 40000 test/GCP/ --exit", - "test-routes": "mocha -t 40000 test/routes/ --exit", - "test": "mocha -t 40000 test/ --exit", - "test-debug": "_mocha -t 40000 test/ --exit" - }, - "author": "", - "mocha": { - "recursive": true, - "timeout": 40000 - } + "name": "Test-rawnode", + "version": "0.1.0", + "description": "Test-rawnode", + "main": "tests.js", + "private": true, + "repository": "", + "keywords": [ + "test" + ], + "scripts": { + "test-aws": "AWS_ON_AIR=true mocha -t 40000 test/ --exit", + "test-gcp": "mocha -t 40000 test/GCP/ --exit", + "test-routes": "mocha -t 40000 test/routes/ --exit", + "test": "mocha -t 40000 test/ --exit", + "test-debug": "_mocha -t 40000 test/ --exit" + }, + "author": "", + "mocha": { + "recursive": true, + "timeout": 40000 + } } diff --git a/tests/functional/raw-node/test/GCP/README.MD b/tests/functional/raw-node/test/GCP/README.MD index bfc58a616a..6aecd181ac 100644 --- a/tests/functional/raw-node/test/GCP/README.MD +++ b/tests/functional/raw-node/test/GCP/README.MD @@ -1,5 +1,5 @@ -This directory contains GCP API functional tests. +This directory contains GCP API functional tests. These tests will verify that the GCP API implementation located in `lib/data/external/GCP` behaves as intended: correct error responses, edge case -handling, and correct responses. \ No newline at end of file +handling, and correct responses. diff --git a/tests/functional/raw-node/test/GCP/bucket/bucket.js b/tests/functional/raw-node/test/GCP/bucket/bucket.js index f155a0e0b8..8f33b2c688 100644 --- a/tests/functional/raw-node/test/GCP/bucket/bucket.js +++ b/tests/functional/raw-node/test/GCP/bucket/bucket.js @@ -12,8 +12,7 @@ const { } = require('@aws-sdk/client-s3'); const { GCP } = arsenal.storage.data.external.GCP; const { genUniqID, genBucketName, gcpRetry } = require('../../../utils/gcpUtils'); -const { getRealAwsConfig } = - require('../../../../aws-node-sdk/test/support/awsConfig'); +const { getRealAwsConfig } = require('../../../../aws-node-sdk/test/support/awsConfig'); const { listingHardLimit } = require('../../../../../../constants'); const credentialOne = 'gcpbackend'; @@ -54,10 +53,7 @@ describe('GCP: Bucket', function testSuite() { const { $metadata, ...data } = res; assert.strictEqual($metadata?.httpStatusCode, 200); // Ensure MetaVersionId is present and non-empty - assert.ok( - typeof data.MetaVersionId === 'string' - && data.MetaVersionId.length > 0 - ); + assert.ok(typeof data.MetaVersionId === 'string' && data.MetaVersionId.length > 0); }); }); @@ -66,81 +62,82 @@ describe('GCP: Bucket', function testSuite() { const bigSize = listingHardLimit + 1; function populateBucket(createdObjects, callback) { - process.stdout.write( - `Putting ${createdObjects.length} objects into bucket\n`); + process.stdout.write(`Putting ${createdObjects.length} objects into bucket\n`); async.mapLimit( createdObjects, 10, - async object => gcpClient.send(new PutObjectCommand({ - Bucket: bucketName, - Key: object, - })), + async object => + gcpClient.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: object, + }), + ), err => { if (err) { process.stdout.write(`err putting objects ${err}\n`); } return callback(err); - } + }, ); } function removeObjects(createdObjects, callback) { - process.stdout.write( - `Deleting ${createdObjects.length} objects from bucket\n`); + process.stdout.write(`Deleting ${createdObjects.length} objects from bucket\n`); async.mapLimit( createdObjects, 10, - async object => gcpClient.send(new DeleteObjectCommand({ - Bucket: bucketName, - Key: object, - })), + async object => + gcpClient.send( + new DeleteObjectCommand({ + Bucket: bucketName, + Key: object, + }), + ), err => { if (err) { process.stdout.write(`err deleting objects ${err}\n`); } return callback(err); - } + }, ); } it('should return 200', async () => { - const res = await gcpClient.send( - new ListObjectsCommand({ Bucket: bucketName })); + const res = await gcpClient.send(new ListObjectsCommand({ Bucket: bucketName })); assert.strictEqual(res.$metadata?.httpStatusCode, 200); }); describe('with less than listingHardLimit number of objects', () => { - const createdObjects = Array.from( - Array(smallSize).keys()).map(i => `someObject-${i}`); + const createdObjects = Array.from(Array(smallSize).keys()).map(i => `someObject-${i}`); before(done => populateBucket(createdObjects, done)); after(done => removeObjects(createdObjects, done)); it(`should list all ${smallSize} created objects`, async () => { - const res = await gcpClient.send( - new ListObjectsCommand({ Bucket: bucketName })); + const res = await gcpClient.send(new ListObjectsCommand({ Bucket: bucketName })); assert.strictEqual(res.Contents.length, smallSize); }); it('should list MaxKeys number of objects with MaxKeys at 10', async () => { - const res = await gcpClient.send(new ListObjectsCommand({ - Bucket: bucketName, - MaxKeys: 10, - })); + const res = await gcpClient.send( + new ListObjectsCommand({ + Bucket: bucketName, + MaxKeys: 10, + }), + ); assert.strictEqual(res.Contents.length, 10); }); }); describe('with more than listingHardLimit number of objects', () => { - const createdObjects = Array.from( - Array(bigSize).keys()).map(i => `someObject-${i}`); + const createdObjects = Array.from(Array(bigSize).keys()).map(i => `someObject-${i}`); before(done => populateBucket(createdObjects, done)); after(done => removeObjects(createdObjects, done)); it('should list at max 1000 of objects created', async () => { - const res = await gcpClient.send( - new ListObjectsCommand({ Bucket: bucketName })); + const res = await gcpClient.send(new ListObjectsCommand({ Bucket: bucketName })); assert.strictEqual(res.Contents.length, listingHardLimit); }); @@ -157,10 +154,12 @@ describe('GCP: Bucket', function testSuite() { // Actual behavior: it returns a list longer than 1000 objects when // max-keys is greater than 1000 it.skip('should list at max 1000, ignoring MaxKeys', async () => { - const res = await gcpClient.send(new ListObjectsCommand({ - Bucket: bucketName, - MaxKeys: 1001, - })); + const res = await gcpClient.send( + new ListObjectsCommand({ + Bucket: bucketName, + MaxKeys: 1001, + }), + ); assert.strictEqual(res.Contents.length, listingHardLimit); }); }); diff --git a/tests/functional/raw-node/test/GCP/bucket/versioning.js b/tests/functional/raw-node/test/GCP/bucket/versioning.js index 45163787a4..1ae5d5afe7 100644 --- a/tests/functional/raw-node/test/GCP/bucket/versioning.js +++ b/tests/functional/raw-node/test/GCP/bucket/versioning.js @@ -8,8 +8,7 @@ const { } = require('@aws-sdk/client-s3'); const { GCP } = arsenal.storage.data.external.GCP; const { genBucketName, gcpRetry } = require('../../../utils/gcpUtils'); -const { getRealAwsConfig } = - require('../../../../aws-node-sdk/test/support/awsConfig'); +const { getRealAwsConfig } = require('../../../../aws-node-sdk/test/support/awsConfig'); const credentialOne = 'gcpbackend'; const config = getRealAwsConfig(credentialOne); @@ -28,24 +27,26 @@ describe('GCP: Bucket Versioning', function testSuite() { }); it('should enable bucket versioning', async () => { - await gcpClient.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: { Status: 'Enabled' }, - })); - - const res = await gcpClient.send( - new GetBucketVersioningCommand({ Bucket: bucketName })); + await gcpClient.send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ); + + const res = await gcpClient.send(new GetBucketVersioningCommand({ Bucket: bucketName })); assert.strictEqual(res.Status, 'Enabled'); }); it('should disable bucket versioning', async () => { - await gcpClient.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: { Status: 'Suspended' }, - })); - - const res = await gcpClient.send( - new GetBucketVersioningCommand({ Bucket: bucketName })); + await gcpClient.send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: { Status: 'Suspended' }, + }), + ); + + const res = await gcpClient.send(new GetBucketVersioningCommand({ Bucket: bucketName })); assert.strictEqual(res.Status, 'Suspended'); }); }); diff --git a/tests/functional/raw-node/test/GCP/object/completeMpu.js b/tests/functional/raw-node/test/GCP/object/completeMpu.js index c839985ffa..d70bf58662 100644 --- a/tests/functional/raw-node/test/GCP/object/completeMpu.js +++ b/tests/functional/raw-node/test/GCP/object/completeMpu.js @@ -4,19 +4,9 @@ const arsenal = require('arsenal'); const { promisify } = require('util'); const { ListObjectsCommand } = require('@aws-sdk/client-s3'); const { GCP, GcpUtils } = arsenal.storage.data.external.GCP; -const { - gcpMpuSetup, - genUniqID, - genBucketName, - gcpRetry, - waitForBucketReady, -} = require('../../../utils/gcpUtils'); -const { getRealAwsConfig } = - require('../../../../aws-node-sdk/test/support/awsConfig'); -const { - CreateBucketCommand, - DeleteBucketCommand, -} = require('@aws-sdk/client-s3'); +const { gcpMpuSetup, genUniqID, genBucketName, gcpRetry, waitForBucketReady } = require('../../../utils/gcpUtils'); +const { getRealAwsConfig } = require('../../../../aws-node-sdk/test/support/awsConfig'); +const { CreateBucketCommand, DeleteBucketCommand } = require('@aws-sdk/client-s3'); const credentialOne = 'gcpbackend'; const bucketNames = { @@ -56,7 +46,8 @@ function listObjectsPaginated(gcpClient, bucketName, cb) { } const command = new ListObjectsCommand(params); - return gcpClient.send(command) + return gcpClient + .send(command) .then(res => { const contents = (res && res.Contents) || []; objects.push(...contents); @@ -67,8 +58,7 @@ function listObjectsPaginated(gcpClient, bucketName, cb) { } // AWS listObjects(V1) pagination: prefer NextMarker, fallback to last key. - marker = (res && res.NextMarker) || - (contents.length ? contents[contents.length - 1].Key : undefined); + marker = (res && res.NextMarker) || (contents.length ? contents[contents.length - 1].Key : undefined); if (!marker) { return cb(null, objects); @@ -87,13 +77,18 @@ function emptyBucket(gcpClient, bucketName, cb) { if (err) { return cb(err); } - return async.eachLimit(objects, 20, (object, next) => { - const deleteParams = { - Bucket: bucketName, - Key: object.Key, - }; - return gcpClient.deleteObject(deleteParams, next); - }, cb); + return async.eachLimit( + objects, + 20, + (object, next) => { + const deleteParams = { + Bucket: bucketName, + Key: object.Key, + }; + return gcpClient.deleteObject(deleteParams, next); + }, + cb, + ); }); } @@ -106,41 +101,39 @@ describe('GCP: Complete MPU', function testSuite() { config = getRealAwsConfig(credentialOne); gcpClient = new GCP(config); const buckets = Object.values(bucketNames); - await async.eachSeries( - buckets, - async bucket => { - const cmd = new CreateBucketCommand({ Bucket: bucket.Name }); - await gcpRetry(gcpClient, cmd); - await waitForBucketReady(gcpClient, bucket.Name); - }, - ); + await async.eachSeries(buckets, async bucket => { + const cmd = new CreateBucketCommand({ Bucket: bucket.Name }); + await gcpRetry(gcpClient, cmd); + await waitForBucketReady(gcpClient, bucket.Name); + }); }); after(async () => { const buckets = Object.values(bucketNames); - await async.eachSeries( - buckets, - async bucket => { - await promisify(emptyBucket)(gcpClient, bucket.Name); - const cmd = new DeleteBucketCommand({ Bucket: bucket.Name }); - await gcpRetry(gcpClient, cmd); - }, - ); + await async.eachSeries(buckets, async bucket => { + await promisify(emptyBucket)(gcpClient, bucket.Name); + const cmd = new DeleteBucketCommand({ Bucket: bucket.Name }); + await gcpRetry(gcpClient, cmd); + }); }); describe('when MPU has 0 parts', () => { beforeEach(function beforeFn(done) { this.currentTest.key = `somekey-${genUniqID()}`; - gcpMpuSetupWrapper.call(this, { - gcpClient, - bucketNames, - key: this.currentTest.key, - partCount: 0, partSize, - }, done); + gcpMpuSetupWrapper.call( + this, + { + gcpClient, + bucketNames, + key: this.currentTest.key, + partCount: 0, + partSize, + }, + done, + ); }); - it('should return error if 0 parts are given in MPU complete', - function testFn(done) { + it('should return error if 0 parts are given in MPU complete', function testFn(done) { const params = { Bucket: bucketNames.main.Name, MPU: bucketNames.mpu.Name, @@ -159,20 +152,28 @@ describe('GCP: Complete MPU', function testSuite() { describe('when MPU has 1 uploaded part', () => { beforeEach(function beforeFn(done) { this.currentTest.key = `somekey-${genUniqID()}`; - gcpMpuSetupWrapper.call(this, { - gcpClient, - bucketNames, - key: this.currentTest.key, - partCount: 1, partSize, - }, done); + gcpMpuSetupWrapper.call( + this, + { + gcpClient, + bucketNames, + key: this.currentTest.key, + partCount: 1, + partSize, + }, + done, + ); }); - it('should successfully complete MPU', - function testFn(done) { - const parts = GcpUtils.createMpuList({ - Key: this.test.key, - UploadId: this.test.uploadId, - }, 'parts', 1).map(item => { + it('should successfully complete MPU', function testFn(done) { + const parts = GcpUtils.createMpuList( + { + Key: this.test.key, + UploadId: this.test.uploadId, + }, + 'parts', + 1, + ).map(item => { Object.assign(item, { ETag: this.test.etagList[item.PartNumber - 1], }); @@ -186,8 +187,7 @@ describe('GCP: Complete MPU', function testSuite() { MultipartUpload: { Parts: parts }, }; gcpClient.completeMultipartUpload(params, (err, res) => { - assert.equal(err, null, - `Expected success, but got error ${err}`); + assert.equal(err, null, `Expected success, but got error ${err}`); assert.strictEqual(res.ETag, `"${smallMD5}"`); return done(); }); @@ -197,22 +197,30 @@ describe('GCP: Complete MPU', function testSuite() { describe('when MPU has 1024 uploaded parts', () => { beforeEach(function beforeFn(done) { this.currentTest.key = `somekey-${genUniqID()}`; - gcpMpuSetupWrapper.call(this, { - gcpClient, - bucketNames, - key: this.currentTest.key, - partCount: numParts, partSize, - }, done); + gcpMpuSetupWrapper.call( + this, + { + gcpClient, + bucketNames, + key: this.currentTest.key, + partCount: numParts, + partSize, + }, + done, + ); }); - it('should successfully complete MPU', - function testFn(done) { + it('should successfully complete MPU', function testFn(done) { this.retries(1); - const parts = GcpUtils.createMpuList({ - Key: this.test.key, - UploadId: this.test.uploadId, - }, 'parts', numParts).map(item => { + const parts = GcpUtils.createMpuList( + { + Key: this.test.key, + UploadId: this.test.uploadId, + }, + 'parts', + numParts, + ).map(item => { Object.assign(item, { ETag: this.test.etagList[item.PartNumber - 1], }); @@ -226,8 +234,7 @@ describe('GCP: Complete MPU', function testSuite() { MultipartUpload: { Parts: parts }, }; gcpClient.completeMultipartUpload(params, (err, res) => { - assert.equal(err, null, - `Expected success, but got error ${err}`); + assert.equal(err, null, `Expected success, but got error ${err}`); assert.strictEqual(res.ETag, `"${bigMD5}"`); return done(); }); diff --git a/tests/functional/raw-node/test/GCP/object/deleteMpu.js b/tests/functional/raw-node/test/GCP/object/deleteMpu.js index e4accccd6c..ef1310a81c 100644 --- a/tests/functional/raw-node/test/GCP/object/deleteMpu.js +++ b/tests/functional/raw-node/test/GCP/object/deleteMpu.js @@ -2,15 +2,9 @@ const assert = require('assert'); const async = require('async'); const arsenal = require('arsenal'); const { GCP } = arsenal.storage.data.external.GCP; -const { gcpMpuSetup, genUniqID, genBucketName, gcpRetry, waitForBucketReady } = - require('../../../utils/gcpUtils'); -const { getRealAwsConfig } = - require('../../../../aws-node-sdk/test/support/awsConfig'); -const { - CreateBucketCommand, - DeleteBucketCommand, - ListObjectsCommand, -} = require('@aws-sdk/client-s3'); +const { gcpMpuSetup, genUniqID, genBucketName, gcpRetry, waitForBucketReady } = require('../../../utils/gcpUtils'); +const { getRealAwsConfig } = require('../../../../aws-node-sdk/test/support/awsConfig'); +const { CreateBucketCommand, DeleteBucketCommand, ListObjectsCommand } = require('@aws-sdk/client-s3'); const credentialOne = 'gcpbackend'; const bucketNames = { @@ -26,8 +20,7 @@ const partSize = 10; function gcpMpuSetupWrapper(params, callback) { gcpMpuSetup(params, (err, result) => { - assert.equal(err, null, - `Unable to setup MPU test, error ${err}`); + assert.equal(err, null, `Unable to setup MPU test, error ${err}`); const { uploadId, etagList } = result; this.currentTest.uploadId = uploadId; this.currentTest.etagList = etagList; @@ -45,122 +38,128 @@ describe('GCP: Abort MPU', function testSuite() { gcpClient = new GCP(config); const buckets = Object.values(bucketNames); - await async.eachSeries( - buckets, - async bucket => { - await gcpRetry( - gcpClient, - new CreateBucketCommand({ Bucket: bucket.Name }), - ); - await waitForBucketReady(gcpClient, bucket.Name); - }, - ); + await async.eachSeries(buckets, async bucket => { + await gcpRetry(gcpClient, new CreateBucketCommand({ Bucket: bucket.Name })); + await waitForBucketReady(gcpClient, bucket.Name); + }); }); after(async () => { const buckets = Object.values(bucketNames); - await async.eachSeries( - buckets, - async bucket => { - const listCmd = new ListObjectsCommand({ + await async.eachSeries(buckets, async bucket => { + const listCmd = new ListObjectsCommand({ + Bucket: bucket.Name, + }); + const listRes = await gcpClient.send(listCmd); + await async.map(listRes.Contents || [], async object => { + await gcpClient.deleteObject({ Bucket: bucket.Name, + Key: object.Key, }); - const listRes = await gcpClient.send(listCmd); - await async.map(listRes.Contents || [], async object => { - await gcpClient.deleteObject({ - Bucket: bucket.Name, - Key: object.Key, - }); - }); - await gcpRetry( - gcpClient, - new DeleteBucketCommand({ Bucket: bucket.Name }), - ); - }, - ); + }); + await gcpRetry(gcpClient, new DeleteBucketCommand({ Bucket: bucket.Name })); + }); }); describe('when MPU has 0 parts', () => { beforeEach(function beforeFn(done) { this.currentTest.key = `somekey-${genUniqID()}`; - gcpMpuSetupWrapper.call(this, { - gcpClient, - bucketNames, - key: this.currentTest.key, - partCount: 0, partSize, - }, done); + gcpMpuSetupWrapper.call( + this, + { + gcpClient, + bucketNames, + key: this.currentTest.key, + partCount: 0, + partSize, + }, + done, + ); }); it('should abort MPU with 0 parts', function testFn(done) { - return async.waterfall([ - next => { - const params = { - Bucket: bucketNames.main.Name, - MPU: bucketNames.mpu.Name, - Key: this.test.key, - UploadId: this.test.uploadId, - }; - gcpClient.abortMultipartUpload(params, err => { - assert.equal(err, null, - `Expected success, but got error ${err}`); - return next(); - }); - }, - next => { - const keyName = - `${this.test.key}-${this.test.uploadId}/init`; - gcpClient.headObject({ - Bucket: bucketNames.mpu.Name, - Key: keyName, - }, err => { - assert(err); - assert.strictEqual(err.$metadata.httpStatusCode, 404); - return next(); - }); - }, - ], done); + return async.waterfall( + [ + next => { + const params = { + Bucket: bucketNames.main.Name, + MPU: bucketNames.mpu.Name, + Key: this.test.key, + UploadId: this.test.uploadId, + }; + gcpClient.abortMultipartUpload(params, err => { + assert.equal(err, null, `Expected success, but got error ${err}`); + return next(); + }); + }, + next => { + const keyName = `${this.test.key}-${this.test.uploadId}/init`; + gcpClient.headObject( + { + Bucket: bucketNames.mpu.Name, + Key: keyName, + }, + err => { + assert(err); + assert.strictEqual(err.$metadata.httpStatusCode, 404); + return next(); + }, + ); + }, + ], + done, + ); }); }); describe('when MPU is incomplete', () => { beforeEach(function beforeFn(done) { this.currentTest.key = `somekey-${genUniqID()}`; - gcpMpuSetupWrapper.call(this, { - gcpClient, - bucketNames, - key: this.currentTest.key, - partCount: numParts, partSize, - }, done); + gcpMpuSetupWrapper.call( + this, + { + gcpClient, + bucketNames, + key: this.currentTest.key, + partCount: numParts, + partSize, + }, + done, + ); }); it('should abort incomplete MPU', function testFn(done) { - return async.waterfall([ - next => { - const params = { - Bucket: bucketNames.main.Name, - MPU: bucketNames.mpu.Name, - Key: this.test.key, - UploadId: this.test.uploadId, - }; - gcpClient.abortMultipartUpload(params, err => { - assert.equal(err, null, - `Expected success, but got error ${err}`); - return next(); - }); - }, - next => { - const keyName = - `${this.test.key}-${this.test.uploadId}/init`; - gcpClient.headObject({ - Bucket: bucketNames.mpu.Name, - Key: keyName, - }, err => { - assert(err); - assert.strictEqual(err.$metadata.httpStatusCode, 404); - return next(); - }); - }, - ], err => done(err)); + return async.waterfall( + [ + next => { + const params = { + Bucket: bucketNames.main.Name, + MPU: bucketNames.mpu.Name, + Key: this.test.key, + UploadId: this.test.uploadId, + }; + gcpClient.abortMultipartUpload(params, err => { + assert.equal(err, null, `Expected success, but got error ${err}`); + return next(); + }); + }, + next => { + const keyName = `${this.test.key}-${this.test.uploadId}/init`; + gcpClient.headObject( + { + Bucket: bucketNames.mpu.Name, + Key: keyName, + }, + err => { + assert(err); + assert.strictEqual(err.$metadata.httpStatusCode, 404); + return next(); + }, + ); + }, + ], + err => done(err), + ); }); }); }); diff --git a/tests/functional/raw-node/test/GCP/object/initiateMpu.js b/tests/functional/raw-node/test/GCP/object/initiateMpu.js index 95349f3ade..31587eee7c 100644 --- a/tests/functional/raw-node/test/GCP/object/initiateMpu.js +++ b/tests/functional/raw-node/test/GCP/object/initiateMpu.js @@ -9,12 +9,8 @@ const { gcpCreateMultipartUploadWithRetry, waitForBucketReady, } = require('../../../utils/gcpUtils'); -const { getRealAwsConfig } = - require('../../../../aws-node-sdk/test/support/awsConfig'); -const { - CreateBucketCommand, - DeleteBucketCommand, -} = require('@aws-sdk/client-s3'); +const { getRealAwsConfig } = require('../../../../aws-node-sdk/test/support/awsConfig'); +const { CreateBucketCommand, DeleteBucketCommand } = require('@aws-sdk/client-s3'); const credentialOne = 'gcpbackend'; const bucketNames = { @@ -35,29 +31,17 @@ describe('GCP: Initiate MPU', function testSuite() { config = getRealAwsConfig(credentialOne); gcpClient = new GCP(config); const buckets = Object.values(bucketNames); - await async.eachSeries( - buckets, - async bucket => { - await gcpRetry( - gcpClient, - new CreateBucketCommand({ Bucket: bucket.Name }), - ); - await waitForBucketReady(gcpClient, bucket.Name); - }, - ); + await async.eachSeries(buckets, async bucket => { + await gcpRetry(gcpClient, new CreateBucketCommand({ Bucket: bucket.Name })); + await waitForBucketReady(gcpClient, bucket.Name); + }); }); after(async () => { const buckets = Object.values(bucketNames); - await async.eachSeries( - buckets, - async bucket => { - await gcpRetry( - gcpClient, - new DeleteBucketCommand({ Bucket: bucket.Name }), - ); - }, - ); + await async.eachSeries(buckets, async bucket => { + await gcpRetry(gcpClient, new DeleteBucketCommand({ Bucket: bucket.Name })); + }); }); it('Should create a multipart upload object', async () => { @@ -72,32 +56,37 @@ describe('GCP: Initiate MPU', function testSuite() { const mpuInitKey = `${keyName}-${createRes.UploadId}/init`; const headRes = await new Promise((resolve, reject) => { - gcpClient.headObject({ - Bucket: bucketNames.mpu.Name, - Key: mpuInitKey, - }, (err, res) => { - if (err) { - process.stdout - .write(`err in retrieving object ${err}`); - return reject(err); - } - return resolve(res); - }); + gcpClient.headObject( + { + Bucket: bucketNames.mpu.Name, + Key: mpuInitKey, + }, + (err, res) => { + if (err) { + process.stdout.write(`err in retrieving object ${err}`); + return reject(err); + } + return resolve(res); + }, + ); }); assert.strictEqual(headRes.Metadata.special, specialKey); await new Promise((resolve, reject) => { - gcpClient.abortMultipartUpload({ - Bucket: bucketNames.main.Name, - MPU: bucketNames.mpu.Name, - UploadId: createRes.UploadId, - Key: keyName, - }, err => { - if (err) { - return reject(err); - } - return resolve(); - }); + gcpClient.abortMultipartUpload( + { + Bucket: bucketNames.main.Name, + MPU: bucketNames.mpu.Name, + UploadId: createRes.UploadId, + Key: keyName, + }, + err => { + if (err) { + return reject(err); + } + return resolve(); + }, + ); }); }); }); diff --git a/tests/functional/raw-node/test/GCP/object/object.js b/tests/functional/raw-node/test/GCP/object/object.js index 5cc3e445b1..e94d400728 100644 --- a/tests/functional/raw-node/test/GCP/object/object.js +++ b/tests/functional/raw-node/test/GCP/object/object.js @@ -3,8 +3,7 @@ const async = require('async'); const arsenal = require('arsenal'); const { GCP } = arsenal.storage.data.external.GCP; const { genUniqID, genBucketName, gcpRetry } = require('../../../utils/gcpUtils'); -const { getRealAwsConfig } = - require('../../../../aws-node-sdk/test/support/awsConfig'); +const { getRealAwsConfig } = require('../../../../aws-node-sdk/test/support/awsConfig'); const { CreateBucketCommand, DeleteBucketCommand, @@ -23,10 +22,7 @@ describe('GCP: Object', function testSuite() { const bucketName = genBucketName('object'); before(async () => { - await gcpRetry( - gcpClient, - new CreateBucketCommand({ Bucket: bucketName }), - ); + await gcpRetry(gcpClient, new CreateBucketCommand({ Bucket: bucketName })); }); after(async () => { @@ -36,10 +32,12 @@ describe('GCP: Object', function testSuite() { async function setupExistingObject(test) { /* eslint-disable no-param-reassign */ test.key = `somekey-${genUniqID()}`; - const res = await gcpClient.send(new PutObjectCommand({ - Bucket: bucketName, - Key: test.key, - })); + const res = await gcpClient.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: test.key, + }), + ); test.uploadId = res.VersionId; test.ETag = res.ETag; /* eslint-enable no-param-reassign */ @@ -49,10 +47,12 @@ describe('GCP: Object', function testSuite() { if (!test.key) { return; } - await gcpClient.send(new DeleteObjectCommand({ - Bucket: bucketName, - Key: test.key, - })); + await gcpClient.send( + new DeleteObjectCommand({ + Bucket: bucketName, + Key: test.key, + }), + ); } describe('HEAD Object', () => { @@ -66,10 +66,12 @@ describe('GCP: Object', function testSuite() { }); it('should successfully retrieve object', async function testFn() { - const res = await gcpClient.send(new HeadObjectCommand({ - Bucket: bucketName, - Key: this.test.key, - })); + const res = await gcpClient.send( + new HeadObjectCommand({ + Bucket: bucketName, + Key: this.test.key, + }), + ); assert.strictEqual(res.ETag, this.test.ETag); assert.ok(res.$metadata && res.$metadata.httpStatusCode === 200); }); @@ -79,14 +81,17 @@ describe('GCP: Object', function testSuite() { it('should return 404', async () => { const badObjectKey = `nonexistingkey-${genUniqID()}`; await new Promise(resolve => { - gcpClient.headObject({ - Bucket: bucketName, - Key: badObjectKey, - }, err => { - assert(err); - assert.strictEqual(err.$metadata.httpStatusCode, 404); - resolve(); - }); + gcpClient.headObject( + { + Bucket: bucketName, + Key: badObjectKey, + }, + err => { + assert(err); + assert.strictEqual(err.$metadata.httpStatusCode, 404); + resolve(); + }, + ); }); }); }); @@ -103,10 +108,12 @@ describe('GCP: Object', function testSuite() { }); it('should successfully retrieve object', async function testFn() { - const res = await gcpClient.send(new GetObjectCommand({ - Bucket: bucketName, - Key: this.test.key, - })); + const res = await gcpClient.send( + new GetObjectCommand({ + Bucket: bucketName, + Key: this.test.key, + }), + ); assert.strictEqual(res.ETag, this.test.ETag); assert.strictEqual(res.VersionId, this.test.uploadId); }); @@ -115,15 +122,18 @@ describe('GCP: Object', function testSuite() { describe('without existing object in bucket', () => { it('should return 404 and NoSuchKey', done => { const badObjectKey = `nonexistingkey-${genUniqID()}`; - gcpClient.getObject({ - Bucket: bucketName, - Key: badObjectKey, - }, err => { - assert(err); - assert.strictEqual(err.$metadata?.httpStatusCode, 404); - assert.strictEqual(err.name, 'NoSuchKey'); - return done(); - }); + gcpClient.getObject( + { + Bucket: bucketName, + Key: badObjectKey, + }, + err => { + assert(err); + assert.strictEqual(err.$metadata?.httpStatusCode, 404); + assert.strictEqual(err.name, 'NoSuchKey'); + return done(); + }, + ); }); }); }); @@ -139,35 +149,42 @@ describe('GCP: Object', function testSuite() { }); it('should overwrite object', function testFn(done) { - gcpClient.putObject({ - Bucket: bucketName, - Key: this.test.key, - }, (err, res) => { - assert.notStrictEqual(res.VersionId, this.test.uploadId); - return done(); - }); + gcpClient.putObject( + { + Bucket: bucketName, + Key: this.test.key, + }, + (err, res) => { + assert.notStrictEqual(res.VersionId, this.test.uploadId); + return done(); + }, + ); }); }); describe('without existing object in bucket', () => { it('should successfully put object', function testFn(done) { this.test.key = `somekey-${genUniqID()}`; - gcpClient.putObject({ - Bucket: bucketName, - Key: this.test.key, - }, (err, putRes) => { - assert.equal(err, null, - `Expected success, got error ${err}`); - gcpClient.getObject({ + gcpClient.putObject( + { Bucket: bucketName, Key: this.test.key, - }, (getErr, getRes) => { - assert.equal(getErr, null, - `Expected success, got error ${getErr}`); - assert.strictEqual(getRes.VersionId, putRes.VersionId); - return done(); - }); - }); + }, + (err, putRes) => { + assert.equal(err, null, `Expected success, got error ${err}`); + gcpClient.getObject( + { + Bucket: bucketName, + Key: this.test.key, + }, + (getErr, getRes) => { + assert.equal(getErr, null, `Expected success, got error ${getErr}`); + assert.strictEqual(getRes.VersionId, putRes.VersionId); + return done(); + }, + ); + }, + ); }); }); }); @@ -178,73 +195,88 @@ describe('GCP: Object', function testSuite() { describe('with existing object in bucket', () => { beforeEach(async () => { - await gcpClient.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectKey, - })); - }); - - it('should successfully delete object', done => { - async.waterfall([ - next => gcpClient.deleteObject({ + await gcpClient.send( + new PutObjectCommand({ Bucket: bucketName, Key: objectKey, - }, err => { - assert.equal(err, null, - `Expected success, got error ${err}`); - return next(); }), - next => { - gcpClient.send(new GetObjectCommand({ - Bucket: bucketName, - Key: objectKey, - })) - .then(() => { - assert.fail('Expected NoSuchKey error'); - }) - .catch(err => { - assert(err); - assert.strictEqual( - err.$metadata && err.$metadata.httpStatusCode, - 404); - assert.strictEqual(err.name, 'NoSuchKey'); - return next(); - }); - }, - ], err => done(err)); + ); + }); + + it('should successfully delete object', done => { + async.waterfall( + [ + next => + gcpClient.deleteObject( + { + Bucket: bucketName, + Key: objectKey, + }, + err => { + assert.equal(err, null, `Expected success, got error ${err}`); + return next(); + }, + ), + next => { + gcpClient + .send( + new GetObjectCommand({ + Bucket: bucketName, + Key: objectKey, + }), + ) + .then(() => { + assert.fail('Expected NoSuchKey error'); + }) + .catch(err => { + assert(err); + assert.strictEqual(err.$metadata && err.$metadata.httpStatusCode, 404); + assert.strictEqual(err.name, 'NoSuchKey'); + return next(); + }); + }, + ], + err => done(err), + ); }); }); describe('without existing object in bucket', () => { it('should return 404 and NoSuchKey', done => { - gcpClient.deleteObject({ - Bucket: bucketName, - Key: badObjectKey, - }, err => { - assert(err); - assert.strictEqual(err.$metadata.httpStatusCode, 404); - assert.strictEqual(err.name, 'NoSuchKey'); - return done(); - }); + gcpClient.deleteObject( + { + Bucket: bucketName, + Key: badObjectKey, + }, + err => { + assert(err); + assert.strictEqual(err.$metadata.httpStatusCode, 404); + assert.strictEqual(err.name, 'NoSuchKey'); + return done(); + }, + ); }); }); }); describe('COPY Object', () => { describe('without existing object in bucket', () => { - it('should return 404 and \'NoSuchKey\'', done => { + it("should return 404 and 'NoSuchKey'", done => { const missingObject = `nonexistingkey-${genUniqID()}`; const someKey = `somekey-${genUniqID()}`; - gcpClient.copyObject({ - Bucket: bucketName, - Key: someKey, - CopySource: `/${bucketName}/${missingObject}`, - }, err => { - assert(err); - assert.strictEqual(err.$metadata.httpStatusCode, 404); - assert.strictEqual(err.name, 'NoSuchKey'); - return done(); - }); + gcpClient.copyObject( + { + Bucket: bucketName, + Key: someKey, + CopySource: `/${bucketName}/${missingObject}`, + }, + err => { + assert(err); + assert.strictEqual(err.$metadata.httpStatusCode, 404); + assert.strictEqual(err.name, 'NoSuchKey'); + return done(); + }, + ); }); }); @@ -253,102 +285,128 @@ describe('GCP: Object', function testSuite() { this.currentTest.key = `somekey-${genUniqID()}`; this.currentTest.copyKey = `copykey-${genUniqID()}`; this.currentTest.initValue = `${genUniqID()}`; - const res = await gcpClient.send(new PutObjectCommand({ - Bucket: bucketName, - Key: this.currentTest.copyKey, - Metadata: { - value: this.currentTest.initValue, - }, - })); + const res = await gcpClient.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: this.currentTest.copyKey, + Metadata: { + value: this.currentTest.initValue, + }, + }), + ); this.currentTest.ETag = res.ETag; }); afterEach(function afterFn(done) { - async.parallel([ - next => gcpClient.deleteObject({ - Bucket: bucketName, - Key: this.currentTest.key, - }, err => { - if (err) { - process.stdout.write(`err in deleting object ${err}\n`); - } - return next(err); - }), - next => gcpClient.deleteObject({ - Bucket: bucketName, - Key: this.currentTest.copyKey, - }, err => { - if (err) { - process.stdout - .write(`err in deleting copy object ${err}\n`); - } - return next(err); - }), - ], done); + async.parallel( + [ + next => + gcpClient.deleteObject( + { + Bucket: bucketName, + Key: this.currentTest.key, + }, + err => { + if (err) { + process.stdout.write(`err in deleting object ${err}\n`); + } + return next(err); + }, + ), + next => + gcpClient.deleteObject( + { + Bucket: bucketName, + Key: this.currentTest.copyKey, + }, + err => { + if (err) { + process.stdout.write(`err in deleting copy object ${err}\n`); + } + return next(err); + }, + ), + ], + done, + ); }); - it('should successfully copy with REPLACE directive', - function testFn(done) { + it('should successfully copy with REPLACE directive', function testFn(done) { const newValue = `${genUniqID()}`; - async.waterfall([ - next => gcpClient.copyObject({ - Bucket: bucketName, - Key: this.test.key, - CopySource: `/${bucketName}/${this.test.copyKey}`, - MetadataDirective: 'REPLACE', - Metadata: { - value: newValue, - }, - }, err => { - assert.equal(err, null, - `Expected success, but got error ${err}`); - return next(); - }), - next => gcpClient.headObject({ - Bucket: bucketName, - Key: this.test.key, - }, (err, res) => { - if (err) { - process.stdout - .write(`err in retrieving object ${err}\n`); - return next(err); - } - assert.strictEqual(res.ETag, this.test.ETag); - assert.notStrictEqual(res.Metadata.value, - this.test.initValue); - return next(); - }), - ], done); + async.waterfall( + [ + next => + gcpClient.copyObject( + { + Bucket: bucketName, + Key: this.test.key, + CopySource: `/${bucketName}/${this.test.copyKey}`, + MetadataDirective: 'REPLACE', + Metadata: { + value: newValue, + }, + }, + err => { + assert.equal(err, null, `Expected success, but got error ${err}`); + return next(); + }, + ), + next => + gcpClient.headObject( + { + Bucket: bucketName, + Key: this.test.key, + }, + (err, res) => { + if (err) { + process.stdout.write(`err in retrieving object ${err}\n`); + return next(err); + } + assert.strictEqual(res.ETag, this.test.ETag); + assert.notStrictEqual(res.Metadata.value, this.test.initValue); + return next(); + }, + ), + ], + done, + ); }); - it('should successfully copy with COPY directive', - function testFn(done) { - async.waterfall([ - next => gcpClient.copyObject({ - Bucket: bucketName, - Key: this.test.key, - CopySource: `/${bucketName}/${this.test.copyKey}`, - MetadataDirective: 'COPY', - }, err => { - assert.equal(err, null, - `Expected success, but got error ${err}`); - return next(); - }), - next => gcpClient.headObject({ - Bucket: bucketName, - Key: this.test.key, - }, (err, res) => { - if (err) { - process.stdout - .write(`err in retrieving object ${err}\n`); - return next(err); - } - assert.strictEqual(res.ETag, this.test.ETag); - assert.strictEqual(res.Metadata.value, - this.test.initValue); - return next(); - }), - ], done); + it('should successfully copy with COPY directive', function testFn(done) { + async.waterfall( + [ + next => + gcpClient.copyObject( + { + Bucket: bucketName, + Key: this.test.key, + CopySource: `/${bucketName}/${this.test.copyKey}`, + MetadataDirective: 'COPY', + }, + err => { + assert.equal(err, null, `Expected success, but got error ${err}`); + return next(); + }, + ), + next => + gcpClient.headObject( + { + Bucket: bucketName, + Key: this.test.key, + }, + (err, res) => { + if (err) { + process.stdout.write(`err in retrieving object ${err}\n`); + return next(err); + } + assert.strictEqual(res.ETag, this.test.ETag); + assert.strictEqual(res.Metadata.value, this.test.initValue); + return next(); + }, + ), + ], + done, + ); }); }); }); diff --git a/tests/functional/raw-node/test/GCP/object/tagging.js b/tests/functional/raw-node/test/GCP/object/tagging.js index 50f0d4c9e7..000d0c1034 100644 --- a/tests/functional/raw-node/test/GCP/object/tagging.js +++ b/tests/functional/raw-node/test/GCP/object/tagging.js @@ -2,16 +2,17 @@ const assert = require('assert'); const async = require('async'); const arsenal = require('arsenal'); const { GCP } = arsenal.storage.data.external.GCP; -const { genPutTagObj, genGetTagObj, genDelTagObj, genUniqID, genBucketName, gcpRetry } = - require('../../../utils/gcpUtils'); -const { getRealAwsConfig } = - require('../../../../aws-node-sdk/test/support/awsConfig'); -const { gcpTaggingPrefix } = require('../../../../../../constants'); const { - CreateBucketCommand, - DeleteBucketCommand, - PutObjectCommand, -} = require('@aws-sdk/client-s3'); + genPutTagObj, + genGetTagObj, + genDelTagObj, + genUniqID, + genBucketName, + gcpRetry, +} = require('../../../utils/gcpUtils'); +const { getRealAwsConfig } = require('../../../../aws-node-sdk/test/support/awsConfig'); +const { gcpTaggingPrefix } = require('../../../../../../constants'); +const { CreateBucketCommand, DeleteBucketCommand, PutObjectCommand } = require('@aws-sdk/client-s3'); const credentialOne = 'gcpbackend'; @@ -22,149 +23,162 @@ describe('GCP: Object Tagging', function testSuite() { const bucketName = genBucketName('tagging'); before(async () => { - await gcpRetry( - gcpClient, - new CreateBucketCommand({ Bucket: bucketName }), - ); + await gcpRetry(gcpClient, new CreateBucketCommand({ Bucket: bucketName })); }); after(async () => { - await gcpRetry( - gcpClient, - new DeleteBucketCommand({ Bucket: bucketName }), - ); + await gcpRetry(gcpClient, new DeleteBucketCommand({ Bucket: bucketName })); }); afterEach(function afterFn(done) { - gcpClient.deleteObject({ - Bucket: bucketName, - Key: this.currentTest.key, - }, err => { - if (err) { - process.stdout.write(`err in deleting object ${err}`); - } - return done(err); - }); + gcpClient.deleteObject( + { + Bucket: bucketName, + Key: this.currentTest.key, + }, + err => { + if (err) { + process.stdout.write(`err in deleting object ${err}`); + } + return done(err); + }, + ); }); describe('PUT Object Tagging', () => { beforeEach(async function beforeFn() { this.currentTest.key = `somekey-${genUniqID()}`; this.currentTest.specialKey = `veryspecial-${genUniqID()}`; - const res = await gcpClient.send(new PutObjectCommand({ - Bucket: bucketName, - Key: this.currentTest.key, - })); + const res = await gcpClient.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: this.currentTest.key, + }), + ); this.currentTest.versionId = res.VersionId; }); it('should successfully put object tags', function testFn(done) { - async.waterfall([ - next => gcpClient.putObjectTagging({ - Bucket: bucketName, - Key: this.test.key, - VersionId: this.test.versionId, - Tagging: { - TagSet: [ + async.waterfall( + [ + next => + gcpClient.putObjectTagging( { - Key: this.test.specialKey, - Value: this.test.specialKey, + Bucket: bucketName, + Key: this.test.key, + VersionId: this.test.versionId, + Tagging: { + TagSet: [ + { + Key: this.test.specialKey, + Value: this.test.specialKey, + }, + ], + }, }, - ], - }, - }, err => { - assert.equal(err, null, - `Expected success, got error ${err}`); - return next(); - }), - next => gcpClient.headObject({ - Bucket: bucketName, - Key: this.test.key, - VersionId: this.test.versionId, - }, (err, res) => { - if (err) { - process.stdout.write(`err in retrieving object ${err}`); - return next(err); - } - const metaKey = `${gcpTaggingPrefix}${this.test.specialKey}`; - const toCompare = res.Metadata[metaKey]; - assert.strictEqual(toCompare, this.test.specialKey); - return next(); - }), - ], done); + err => { + assert.equal(err, null, `Expected success, got error ${err}`); + return next(); + }, + ), + next => + gcpClient.headObject( + { + Bucket: bucketName, + Key: this.test.key, + VersionId: this.test.versionId, + }, + (err, res) => { + if (err) { + process.stdout.write(`err in retrieving object ${err}`); + return next(err); + } + const metaKey = `${gcpTaggingPrefix}${this.test.specialKey}`; + const toCompare = res.Metadata[metaKey]; + assert.strictEqual(toCompare, this.test.specialKey); + return next(); + }, + ), + ], + done, + ); }); describe('when tagging parameter is incorrect', () => { - it('should return 400 and BadRequest if more than ' + - '10 tags are given', function testFun(done) { - return gcpClient.putObjectTagging({ - Bucket: bucketName, - Key: this.test.key, - VersionId: this.test.versionId, - Tagging: { - TagSet: genPutTagObj(11), + it('should return 400 and BadRequest if more than ' + '10 tags are given', function testFun(done) { + return gcpClient.putObjectTagging( + { + Bucket: bucketName, + Key: this.test.key, + VersionId: this.test.versionId, + Tagging: { + TagSet: genPutTagObj(11), + }, }, - }, err => { - assert(err); - assert.strictEqual(err.code, 400); - assert.strictEqual(err.message, 'BadRequest'); - return done(); - }); + err => { + assert(err); + assert.strictEqual(err.code, 400); + assert.strictEqual(err.message, 'BadRequest'); + return done(); + }, + ); }); - it('should return 400 and InvalidTag if given duplicate keys', - function testFn(done) { - return gcpClient.putObjectTagging({ - Bucket: bucketName, - Key: this.test.key, - VersionId: this.test.versionId, - Tagging: { - TagSet: genPutTagObj(10, true), + it('should return 400 and InvalidTag if given duplicate keys', function testFn(done) { + return gcpClient.putObjectTagging( + { + Bucket: bucketName, + Key: this.test.key, + VersionId: this.test.versionId, + Tagging: { + TagSet: genPutTagObj(10, true), + }, }, - }, err => { - assert(err); - assert.strictEqual(err.code, 400); - assert.strictEqual(err.message, 'InvalidTag'); - return done(); - }); + err => { + assert(err); + assert.strictEqual(err.code, 400); + assert.strictEqual(err.message, 'InvalidTag'); + return done(); + }, + ); }); - it('should return 400 and InvalidTag if given invalid key', - function testFn(done) { - return gcpClient.putObjectTagging({ - Bucket: bucketName, - Key: this.test.key, - VersionId: this.test.versionId, - Tagging: { - TagSet: [ - { Key: Buffer.alloc(129, 'a'), Value: 'bad tag' }, - ], + it('should return 400 and InvalidTag if given invalid key', function testFn(done) { + return gcpClient.putObjectTagging( + { + Bucket: bucketName, + Key: this.test.key, + VersionId: this.test.versionId, + Tagging: { + TagSet: [{ Key: Buffer.alloc(129, 'a'), Value: 'bad tag' }], + }, }, - }, err => { - assert(err); - assert.strictEqual(err.code, 400); - assert.strictEqual(err.message, 'InvalidTag'); - return done(); - }); + err => { + assert(err); + assert.strictEqual(err.code, 400); + assert.strictEqual(err.message, 'InvalidTag'); + return done(); + }, + ); }); - it('should return 400 and InvalidTag if given invalid value', - function testFn(done) { - return gcpClient.putObjectTagging({ - Bucket: bucketName, - Key: this.test.key, - VersionId: this.test.versionId, - Tagging: { - TagSet: [ - { Key: 'badtag', Value: Buffer.alloc(257, 'a') }, - ], + it('should return 400 and InvalidTag if given invalid value', function testFn(done) { + return gcpClient.putObjectTagging( + { + Bucket: bucketName, + Key: this.test.key, + VersionId: this.test.versionId, + Tagging: { + TagSet: [{ Key: 'badtag', Value: Buffer.alloc(257, 'a') }], + }, }, - }, err => { - assert(err); - assert.strictEqual(err.code, 400); - assert.strictEqual(err.message, 'InvalidTag'); - return done(); - }); + err => { + assert(err); + assert.strictEqual(err.code, 400); + assert.strictEqual(err.message, 'InvalidTag'); + return done(); + }, + ); }); }); }); @@ -175,82 +189,89 @@ describe('GCP: Object Tagging', function testSuite() { beforeEach(async function beforeFn() { this.currentTest.key = `somekey-${genUniqID()}`; this.currentTest.specialKey = `veryspecial-${genUniqID()}`; - const { expectedTagObj } = - genGetTagObj(tagSize, `x-goog-meta-${gcpTaggingPrefix}`); + const { expectedTagObj } = genGetTagObj(tagSize, `x-goog-meta-${gcpTaggingPrefix}`); this.currentTest.tagObj = expectedTagObj; - const res = await gcpClient.send(new PutObjectCommand({ - Bucket: bucketName, - Key: this.currentTest.key, - })); + const res = await gcpClient.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: this.currentTest.key, + }), + ); this.currentTest.versionId = res.VersionId; await new Promise((resolve, reject) => { - gcpClient.putObjectTagging({ - Bucket: bucketName, - Key: this.currentTest.key, - VersionId: this.currentTest.versionId, - Tagging: { - TagSet: this.currentTest.tagObj, + gcpClient.putObjectTagging( + { + Bucket: bucketName, + Key: this.currentTest.key, + VersionId: this.currentTest.versionId, + Tagging: { + TagSet: this.currentTest.tagObj, + }, }, - }, err => { - if (err) { - process.stdout - .write(`err in setting object tags ${err}`); - reject(err); - return; - } - resolve(); - }); + err => { + if (err) { + process.stdout.write(`err in setting object tags ${err}`); + reject(err); + return; + } + resolve(); + }, + ); }); }); it('should successfully get object tags', function testFn(done) { - gcpClient.getObjectTagging({ - Bucket: bucketName, - Key: this.test.key, - VersionId: this.test.versionId, - }, (err, res) => { - assert.equal(err, null, - `Expected success, got error ${err}`); - assert.deepStrictEqual(res.TagSet, this.test.tagObj); - return done(); - }); + gcpClient.getObjectTagging( + { + Bucket: bucketName, + Key: this.test.key, + VersionId: this.test.versionId, + }, + (err, res) => { + assert.equal(err, null, `Expected success, got error ${err}`); + assert.deepStrictEqual(res.TagSet, this.test.tagObj); + return done(); + }, + ); }); }); describe('DELETE Object Tagging', () => { function assertObjectMetaTag(params, callback) { - return gcpClient.headObject({ - Bucket: params.bucket, - Key: params.key, - VersionId: params.versionId, - }, (err, res) => { - if (err) { - process.stdout.write(`err in retrieving object ${err}`); - return callback(err); - } - const resMeta = Object.assign({}, res.Metadata || {}); - const tagRes = {}; - const metaRes = {}; - Object.keys(resMeta).forEach(key => { - if (key.startsWith(gcpTaggingPrefix)) { - tagRes[key] = resMeta[key]; - } else { - metaRes[key] = resMeta[key]; + return gcpClient.headObject( + { + Bucket: params.bucket, + Key: params.key, + VersionId: params.versionId, + }, + (err, res) => { + if (err) { + process.stdout.write(`err in retrieving object ${err}`); + return callback(err); } - }); - assert.deepStrictEqual(params.tag, tagRes); - assert.deepStrictEqual(params.meta, metaRes); - return callback(); - }); + const resMeta = Object.assign({}, res.Metadata || {}); + const tagRes = {}; + const metaRes = {}; + Object.keys(resMeta).forEach(key => { + if (key.startsWith(gcpTaggingPrefix)) { + tagRes[key] = resMeta[key]; + } else { + metaRes[key] = resMeta[key]; + } + }); + assert.deepStrictEqual(params.tag, tagRes); + assert.deepStrictEqual(params.meta, metaRes); + return callback(); + }, + ); } beforeEach(async function beforeFn() { this.currentTest.key = `somekey-${genUniqID()}`; this.currentTest.specialKey = `veryspecial-${genUniqID()}`; - const { expectedTagObj, expectedMetaObj } = - genDelTagObj(10, `x-goog-meta-${gcpTaggingPrefix}`); + const { expectedTagObj, expectedMetaObj } = genDelTagObj(10, `x-goog-meta-${gcpTaggingPrefix}`); const expectedTagMeta = {}; Object.keys(expectedTagObj).forEach(header => { @@ -267,40 +288,56 @@ describe('GCP: Object Tagging', function testSuite() { this.currentTest.expectedTagObj = expectedTagMeta; this.currentTest.expectedMetaObj = expectedMetaMeta; - const res = await gcpClient.send(new PutObjectCommand({ - Bucket: bucketName, - Key: this.currentTest.key, - Metadata: Object.assign({}, expectedTagMeta, expectedMetaMeta), - })); + const res = await gcpClient.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: this.currentTest.key, + Metadata: Object.assign({}, expectedTagMeta, expectedMetaMeta), + }), + ); this.currentTest.versionId = res.VersionId; }); it('should successfully delete object tags', function testFn(done) { - async.waterfall([ - next => assertObjectMetaTag({ - bucket: bucketName, - key: this.test.key, - versionId: this.test.versionId, - meta: this.test.expectedMetaObj, - tag: this.test.expectedTagObj, - }, next), - next => gcpClient.deleteObjectTagging({ - Bucket: bucketName, - Key: this.test.key, - VersionId: this.test.versionId, - }, err => { - assert.equal(err, null, - `Expected success, got error ${err}`); - return next(); - }), - next => assertObjectMetaTag({ - bucket: bucketName, - key: this.test.key, - versionId: this.test.versionId, - meta: this.test.expectedMetaObj, - tag: {}, - }, next), - ], done); + async.waterfall( + [ + next => + assertObjectMetaTag( + { + bucket: bucketName, + key: this.test.key, + versionId: this.test.versionId, + meta: this.test.expectedMetaObj, + tag: this.test.expectedTagObj, + }, + next, + ), + next => + gcpClient.deleteObjectTagging( + { + Bucket: bucketName, + Key: this.test.key, + VersionId: this.test.versionId, + }, + err => { + assert.equal(err, null, `Expected success, got error ${err}`); + return next(); + }, + ), + next => + assertObjectMetaTag( + { + bucket: bucketName, + key: this.test.key, + versionId: this.test.versionId, + meta: this.test.expectedMetaObj, + tag: {}, + }, + next, + ), + ], + done, + ); }); }); }); diff --git a/tests/functional/raw-node/test/GCP/object/upload.js b/tests/functional/raw-node/test/GCP/object/upload.js index 8b4445a296..f9ac187028 100644 --- a/tests/functional/raw-node/test/GCP/object/upload.js +++ b/tests/functional/raw-node/test/GCP/object/upload.js @@ -2,15 +2,15 @@ const assert = require('assert'); const async = require('async'); const arsenal = require('arsenal'); const { GCP } = arsenal.storage.data.external.GCP; -const { genUniqID, genBucketName, gcpRetry, gcpUploadWithRetry, waitForBucketReady } = - require('../../../utils/gcpUtils'); -const { getRealAwsConfig } = - require('../../../../aws-node-sdk/test/support/awsConfig'); const { - CreateBucketCommand, - DeleteBucketCommand, - ListObjectsCommand, -} = require('@aws-sdk/client-s3'); + genUniqID, + genBucketName, + gcpRetry, + gcpUploadWithRetry, + waitForBucketReady, +} = require('../../../utils/gcpUtils'); +const { getRealAwsConfig } = require('../../../../aws-node-sdk/test/support/awsConfig'); +const { CreateBucketCommand, DeleteBucketCommand, ListObjectsCommand } = require('@aws-sdk/client-s3'); const credentialOne = 'gcpbackend'; const bucketNames = { @@ -36,39 +36,27 @@ describe('GCP: Upload Object', function testSuite() { config = getRealAwsConfig(credentialOne); gcpClient = new GCP(config); const buckets = Object.values(bucketNames); - await async.eachSeries( - buckets, - async bucket => { - await gcpRetry( - gcpClient, - new CreateBucketCommand({ Bucket: bucket.Name }), - ); - await waitForBucketReady(gcpClient, bucket.Name); - }, - ); + await async.eachSeries(buckets, async bucket => { + await gcpRetry(gcpClient, new CreateBucketCommand({ Bucket: bucket.Name })); + await waitForBucketReady(gcpClient, bucket.Name); + }); }); after(async () => { const buckets = Object.values(bucketNames); - await async.eachSeries( - buckets, - async bucket => { - const listCmd = new ListObjectsCommand({ + await async.eachSeries(buckets, async bucket => { + const listCmd = new ListObjectsCommand({ + Bucket: bucket.Name, + }); + const listRes = await gcpClient.send(listCmd); + await async.map(listRes.Contents || [], async object => { + await gcpClient.deleteObject({ Bucket: bucket.Name, + Key: object.Key, }); - const listRes = await gcpClient.send(listCmd); - await async.map(listRes.Contents || [], async object => { - await gcpClient.deleteObject({ - Bucket: bucket.Name, - Key: object.Key, - }); - }); - await gcpRetry( - gcpClient, - new DeleteBucketCommand({ Bucket: bucket.Name }), - ); - }, - ); + }); + await gcpRetry(gcpClient, new DeleteBucketCommand({ Bucket: bucket.Name })); + }); }); it('should put an object to GCP', async () => { diff --git a/tests/functional/raw-node/test/badChunkSignatureV4.js b/tests/functional/raw-node/test/badChunkSignatureV4.js index 4a05e2d7d7..9a5b03ec0a 100644 --- a/tests/functional/raw-node/test/badChunkSignatureV4.js +++ b/tests/functional/raw-node/test/badChunkSignatureV4.js @@ -2,14 +2,12 @@ const http = require('http'); const async = require('async'); const assert = require('assert'); -const BucketUtility = - require('../../aws-node-sdk/lib/utility/bucket-util'); +const BucketUtility = require('../../aws-node-sdk/lib/utility/bucket-util'); const HttpRequestAuthV4 = require('../utils/HttpRequestAuthV4'); const config = require('../../config.json'); -const DUMMY_SIGNATURE = - 'baadc0debaadc0debaadc0debaadc0debaadc0debaadc0debaadc0debaadc0de'; +const DUMMY_SIGNATURE = 'baadc0debaadc0debaadc0debaadc0debaadc0debaadc0debaadc0debaadc0de'; http.globalAgent.keepAlive = true; @@ -31,10 +29,7 @@ function createBucket(bucketUtil, cb) { function cleanupBucket(bucketUtil, cb) { const emptyBucket = async.asyncify(bucketUtil.empty.bind(bucketUtil)); const deleteBucket = async.asyncify(bucketUtil.deleteOne.bind(bucketUtil)); - async.series([ - done => emptyBucket(BUCKET, done), - done => deleteBucket(BUCKET, done), - ], cb); + async.series([done => emptyBucket(BUCKET, done), done => deleteBucket(BUCKET, done)], cb); } class HttpChunkedUploadWithBadSignature extends HttpRequestAuthV4 { @@ -60,39 +55,45 @@ class HttpChunkedUploadWithBadSignature extends HttpRequestAuthV4 { function testChunkedPutWithBadSignature(n, alterSignatureChunkId, cb) { const req = new HttpChunkedUploadWithBadSignature( - `http://${config.ipAddress}:${PORT}/${BUCKET}/obj-${n}`, { + `http://${config.ipAddress}:${PORT}/${BUCKET}/obj-${n}`, + { accessKey: config.accessKey, secretKey: config.secretKey, method: 'PUT', headers: { 'content-length': N_DATA_CHUNKS * DATA_CHUNK_SIZE, - 'connection': 'keep-alive', + connection: 'keep-alive', }, alterSignatureChunkId, - }, res => { - if (alterSignatureChunkId >= 0 && - alterSignatureChunkId <= N_DATA_CHUNKS) { + }, + res => { + if (alterSignatureChunkId >= 0 && alterSignatureChunkId <= N_DATA_CHUNKS) { assert.strictEqual(res.statusCode, 403); } else { assert.strictEqual(res.statusCode, 200); } res.on('data', () => {}); res.on('end', cb); - }); + }, + ); req.on('error', err => { assert.ifError(err); }); - async.timesSeries(N_DATA_CHUNKS, (chunkIndex, done) => { - // console.log(`SENDING NEXT CHUNK OF LENGTH ${CHUNK_DATA.length}`); - if (req.write(CHUNK_DATA)) { - process.nextTick(done); - } else { - req.once('drain', done); - } - }, () => { - req.end(); - }); + async.timesSeries( + N_DATA_CHUNKS, + (chunkIndex, done) => { + // console.log(`SENDING NEXT CHUNK OF LENGTH ${CHUNK_DATA.length}`); + if (req.write(CHUNK_DATA)) { + process.nextTick(done); + } else { + req.once('drain', done); + } + }, + () => { + req.end(); + }, + ); } describe('streaming V4 signature with bad chunk signature', () => { @@ -100,26 +101,32 @@ describe('streaming V4 signature with bad chunk signature', () => { before(done => createBucket(bucketUtil, done)); after(done => cleanupBucket(bucketUtil, done)); - it('Cloudserver should be robust against bad signature in streaming ' + - 'payload', function badSignatureInStreamingPayload(cb) { - this.timeout(120000); - async.timesLimit(N_PUTS, 10, (n, done) => { - // multiple test cases depend on the value of - // alterSignatureChunkId: - // alterSignatureChunkId >= 0 && - // alterSignatureChunkId < N_DATA_CHUNKS - // <=> alter the signature of the target data chunk - // alterSignatureChunkId == N_DATA_CHUNKS - // <=> alter the signature of the last empty chunk that - // carries the last payload signature - // alterSignatureChunkId > N_DATA_CHUNKS - // <=> no signature is altered (regular test case) - // By making n go from 0 to nDatachunks+1, we cover all - // above cases. - - const alterSignatureChunkId = ALTER_CHUNK_SIGNATURE ? - (n % (N_DATA_CHUNKS + 2)) : null; - testChunkedPutWithBadSignature(n, alterSignatureChunkId, done); - }, err => cb(err)); - }); + it( + 'Cloudserver should be robust against bad signature in streaming ' + 'payload', + function badSignatureInStreamingPayload(cb) { + this.timeout(120000); + async.timesLimit( + N_PUTS, + 10, + (n, done) => { + // multiple test cases depend on the value of + // alterSignatureChunkId: + // alterSignatureChunkId >= 0 && + // alterSignatureChunkId < N_DATA_CHUNKS + // <=> alter the signature of the target data chunk + // alterSignatureChunkId == N_DATA_CHUNKS + // <=> alter the signature of the last empty chunk that + // carries the last payload signature + // alterSignatureChunkId > N_DATA_CHUNKS + // <=> no signature is altered (regular test case) + // By making n go from 0 to nDatachunks+1, we cover all + // above cases. + + const alterSignatureChunkId = ALTER_CHUNK_SIGNATURE ? n % (N_DATA_CHUNKS + 2) : null; + testChunkedPutWithBadSignature(n, alterSignatureChunkId, done); + }, + err => cb(err), + ); + }, + ); }); diff --git a/tests/functional/raw-node/test/headObject.js b/tests/functional/raw-node/test/headObject.js index c691dafd97..d79d6fc04b 100644 --- a/tests/functional/raw-node/test/headObject.js +++ b/tests/functional/raw-node/test/headObject.js @@ -10,36 +10,45 @@ const bucket = 'rawnodeapibucket'; describe('api tests', () => { before(done => { - makeS3Request({ - method: 'PUT', - authCredentials, - bucket, - }, err => { - assert.ifError(err); - done(); - }); + makeS3Request( + { + method: 'PUT', + authCredentials, + bucket, + }, + err => { + assert.ifError(err); + done(); + }, + ); }); after(done => { - makeS3Request({ - method: 'DELETE', - authCredentials, - bucket, - }, err => { - assert.ifError(err); - done(); - }); + makeS3Request( + { + method: 'DELETE', + authCredentials, + bucket, + }, + err => { + assert.ifError(err); + done(); + }, + ); }); it('should return 405 on headBucket when bucket is empty string', done => { - makeS3Request({ - method: 'HEAD', - authCredentials, - bucket: '', - }, (err, res) => { - assert.ifError(err); - assert.strictEqual(res.statusCode, 405); - return done(); - }); + makeS3Request( + { + method: 'HEAD', + authCredentials, + bucket: '', + }, + (err, res) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, 405); + return done(); + }, + ); }); }); diff --git a/tests/functional/raw-node/test/lifecycle.js b/tests/functional/raw-node/test/lifecycle.js index af145845bb..6d05669364 100644 --- a/tests/functional/raw-node/test/lifecycle.js +++ b/tests/functional/raw-node/test/lifecycle.js @@ -25,109 +25,133 @@ function makeLifeCycleXML(date) { describe('api tests', () => { before(done => { - makeS3Request({ - method: 'PUT', - authCredentials, - bucket, - }, err => { - assert.ifError(err); - done(); - }); + makeS3Request( + { + method: 'PUT', + authCredentials, + bucket, + }, + err => { + assert.ifError(err); + done(); + }, + ); }); after(done => { - makeS3Request({ - method: 'DELETE', - authCredentials, - bucket, - }, err => { - assert.ifError(err); - done(); - }); + makeS3Request( + { + method: 'DELETE', + authCredentials, + bucket, + }, + err => { + assert.ifError(err); + done(); + }, + ); }); it('should accept a lifecycle policy with a date at midnight', done => { - makeS3Request({ - method: 'PUT', - authCredentials, - bucket, - queryObj: { lifecycle: '' }, - requestBody: makeLifeCycleXML('2024-01-08T00:00:00Z'), - }, (err, res) => { - assert.ifError(err); - assert.strictEqual(res.statusCode, 200); - return done(); - }); + makeS3Request( + { + method: 'PUT', + authCredentials, + bucket, + queryObj: { lifecycle: '' }, + requestBody: makeLifeCycleXML('2024-01-08T00:00:00Z'), + }, + (err, res) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, 200); + return done(); + }, + ); }); it('should accept a lifecycle policy with a date at midnight', done => { - makeS3Request({ - method: 'PUT', - authCredentials, - bucket, - queryObj: { lifecycle: '' }, - requestBody: makeLifeCycleXML('2024-01-08T00:00:00'), - }, (err, res) => { - assert.ifError(err); - assert.strictEqual(res.statusCode, 200); - return done(); - }); + makeS3Request( + { + method: 'PUT', + authCredentials, + bucket, + queryObj: { lifecycle: '' }, + requestBody: makeLifeCycleXML('2024-01-08T00:00:00'), + }, + (err, res) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, 200); + return done(); + }, + ); }); it('should accept a lifecycle policy with a date at midnight', done => { - makeS3Request({ - method: 'PUT', - authCredentials, - bucket, - queryObj: { lifecycle: '' }, - requestBody: makeLifeCycleXML('2024-01-08T06:00:00+06:00'), - }, (err, res) => { - assert.ifError(err); - assert.strictEqual(res.statusCode, 200); - return done(); - }); + makeS3Request( + { + method: 'PUT', + authCredentials, + bucket, + queryObj: { lifecycle: '' }, + requestBody: makeLifeCycleXML('2024-01-08T06:00:00+06:00'), + }, + (err, res) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, 200); + return done(); + }, + ); }); it('should reject a lifecycle policy with a date not at midnight', done => { - makeS3Request({ - method: 'PUT', - authCredentials, - bucket, - queryObj: { lifecycle: '' }, - requestBody: makeLifeCycleXML('2024-01-08T12:34:56Z'), - }, err => { - assert.strictEqual(err.code, 'InvalidArgument'); - assert.strictEqual(err.statusCode, 400); - return done(); - }); + makeS3Request( + { + method: 'PUT', + authCredentials, + bucket, + queryObj: { lifecycle: '' }, + requestBody: makeLifeCycleXML('2024-01-08T12:34:56Z'), + }, + err => { + assert.strictEqual(err.code, 'InvalidArgument'); + assert.strictEqual(err.statusCode, 400); + return done(); + }, + ); }); it('should reject a lifecycle policy with an illegal date', done => { - makeS3Request({ - method: 'PUT', - authCredentials, - bucket, - queryObj: { lifecycle: '' }, - requestBody: makeLifeCycleXML('2024-01-08T00:00:00+34:00'), - }, err => { - // This value is catched by AWS during XML parsing - assert(err.code === 'InvalidArgument' || err.code === 'MalformedXML'); - assert.strictEqual(err.statusCode, 400); - return done(); - }); + makeS3Request( + { + method: 'PUT', + authCredentials, + bucket, + queryObj: { lifecycle: '' }, + requestBody: makeLifeCycleXML('2024-01-08T00:00:00+34:00'), + }, + err => { + // This value is catched by AWS during XML parsing + assert(err.code === 'InvalidArgument' || err.code === 'MalformedXML'); + assert.strictEqual(err.statusCode, 400); + return done(); + }, + ); }); it('should reject a lifecycle policy with a date not at midnight', done => { - makeS3Request({ - method: 'PUT', - authCredentials, - bucket, - queryObj: { lifecycle: '' }, - requestBody: makeLifeCycleXML('2024-01-08T00:00:00.123Z'), - }, err => { - assert.strictEqual(err.code, 'InvalidArgument'); - assert.strictEqual(err.statusCode, 400); - return done(); - }); + makeS3Request( + { + method: 'PUT', + authCredentials, + bucket, + queryObj: { lifecycle: '' }, + requestBody: makeLifeCycleXML('2024-01-08T00:00:00.123Z'), + }, + err => { + assert.strictEqual(err.code, 'InvalidArgument'); + assert.strictEqual(err.statusCode, 400); + return done(); + }, + ); }); }); diff --git a/tests/functional/raw-node/test/malformedDateHeader.js b/tests/functional/raw-node/test/malformedDateHeader.js index eb9a1bcedc..256b5a5985 100644 --- a/tests/functional/raw-node/test/malformedDateHeader.js +++ b/tests/functional/raw-node/test/malformedDateHeader.js @@ -12,12 +12,13 @@ describe('malformed Date header:', () => { path: `/${bucket}/${objectKey}`, method: 'GET', headers: { - 'Date': 'BAD_DATE', - 'Authorization': 'AWS4-HMAC-SHA256 Credential=accessKey1/20260211/us-east-1/s3/aws4_request, ' + + Date: 'BAD_DATE', + Authorization: + 'AWS4-HMAC-SHA256 Credential=accessKey1/20260211/us-east-1/s3/aws4_request, ' + 'SignedHeaders=host, Signature=d459d5b2a2395b4c65d8f8aa2729b22c5abb04614fafbd93ab4fe203e76d21a3', 'X-Amz-Content-Sha256': 'fa8d015f89da2a769d1cea7e3bd77a5670d098d7844cda148a40c1304e5b778b', - 'Host': 'localhost:8000' - } + Host: 'localhost:8000', + }, }; const req = http.request(options, res => { @@ -49,12 +50,13 @@ describe('malformed Date header:', () => { method: 'GET', headers: { 'X-Amz-Date': 'BAD_DATE', - 'Authorization': 'AWS4-HMAC-SHA256 Credential=accessKey1/20260211/us-east-1/s3/aws4_request, ' + + Authorization: + 'AWS4-HMAC-SHA256 Credential=accessKey1/20260211/us-east-1/s3/aws4_request, ' + 'SignedHeaders=host;x-amz-date, ' + 'Signature=d459d5b2a2395b4c65d8f8aa2729b22c5abb04614fafbd93ab4fe203e76d21a3', 'X-Amz-Content-Sha256': 'fa8d015f89da2a769d1cea7e3bd77a5670d098d7844cda148a40c1304e5b778b', - 'Host': 'localhost:8000' - } + Host: 'localhost:8000', + }, }; const req = http.request(options, res => { diff --git a/tests/functional/raw-node/test/routes/routeMetadata.js b/tests/functional/raw-node/test/routes/routeMetadata.js index 81b24a2bfa..b80464f727 100644 --- a/tests/functional/raw-node/test/routes/routeMetadata.js +++ b/tests/functional/raw-node/test/routes/routeMetadata.js @@ -1,8 +1,6 @@ const assert = require('assert'); const http = require('http'); -const { CreateBucketCommand, - PutObjectCommand, - DeleteBucketCommand } = require('@aws-sdk/client-s3'); +const { CreateBucketCommand, PutObjectCommand, DeleteBucketCommand } = require('@aws-sdk/client-s3'); const { makeRequest } = require('../../utils/makeRequest'); const MetadataMock = require('../../utils/MetadataMock'); @@ -20,8 +18,7 @@ const metadataAuthCredentials = { }; function makeMetadataRequest(params, callback) { - const { method, headers, authCredentials, - requestBody, queryObj, path } = params; + const { method, headers, authCredentials, requestBody, queryObj, path } = params; const options = { authCredentials, hostname: ipAddress, @@ -37,8 +34,7 @@ function makeMetadataRequest(params, callback) { } describe('metadata routes with metadata', () => { - const bucketUtil = new BucketUtility( - 'default', { signatureVersion: 'v4' }); + const bucketUtil = new BucketUtility('default', { signatureVersion: 'v4' }); const s3 = bucketUtil.s3; const bucket1 = 'bucket1'; @@ -63,84 +59,98 @@ describe('metadata routes with metadata', () => { let httpServer; before(done => { - httpServer = http.createServer( - (req, res) => metadataMock.onRequest(req, res)).listen(9000, done); + httpServer = http.createServer((req, res) => metadataMock.onRequest(req, res)).listen(9000, done); }); after(() => httpServer.close()); } it('should retrieve list of buckets', done => { - makeMetadataRequest({ - method: 'GET', - authCredentials: metadataAuthCredentials, - path: '/_/metadata/admin/raft_sessions/1/bucket', - }, (err, res) => { - assert.ifError(err); - assert.strictEqual(res.statusCode, 200); - assert(res.body); - const expectedArray = [bucket1, 'users..bucket', bucket2]; - const responseArray = JSON.parse(res.body); - - expectedArray.sort(); - responseArray.sort(); - - assert.deepStrictEqual(responseArray, expectedArray); - return done(); - }); + makeMetadataRequest( + { + method: 'GET', + authCredentials: metadataAuthCredentials, + path: '/_/metadata/admin/raft_sessions/1/bucket', + }, + (err, res) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, 200); + assert(res.body); + const expectedArray = [bucket1, 'users..bucket', bucket2]; + const responseArray = JSON.parse(res.body); + + expectedArray.sort(); + responseArray.sort(); + + assert.deepStrictEqual(responseArray, expectedArray); + return done(); + }, + ); }); it('should retrieve list of objects from bucket', done => { - makeMetadataRequest({ - method: 'GET', - authCredentials: metadataAuthCredentials, - path: `/_/metadata/default/bucket/${bucket1}`, - queryObj: { listingType: 'Delimiter' }, - }, (err, res) => { - assert.ifError(err); - assert.strictEqual(res.statusCode, 200); - const body = JSON.parse(res.body); - assert.strictEqual(body.Contents[0].key, 'testobject1'); - return done(); - }); + makeMetadataRequest( + { + method: 'GET', + authCredentials: metadataAuthCredentials, + path: `/_/metadata/default/bucket/${bucket1}`, + queryObj: { listingType: 'Delimiter' }, + }, + (err, res) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, 200); + const body = JSON.parse(res.body); + assert.strictEqual(body.Contents[0].key, 'testobject1'); + return done(); + }, + ); }); it('should retrieve metadata of bucket', done => { - makeMetadataRequest({ - method: 'GET', - authCredentials: metadataAuthCredentials, - path: `/_/metadata/default/attributes/${bucket1}`, - }, (err, res) => { - assert.ifError(err); - assert.strictEqual(res.statusCode, 200); - assert(res.body); - return done(); - }); + makeMetadataRequest( + { + method: 'GET', + authCredentials: metadataAuthCredentials, + path: `/_/metadata/default/attributes/${bucket1}`, + }, + (err, res) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, 200); + assert(res.body); + return done(); + }, + ); }); it('should retrieve metadata of object', done => { - makeMetadataRequest({ - method: 'GET', - authCredentials: metadataAuthCredentials, - path: `/_/metadata/default/bucket/${bucket1}/${keyName}`, - }, (err, res) => { - assert.ifError(err); - assert(res.body); - assert.strictEqual(res.statusCode, 200); - const body = JSON.parse(res.body); - assert(body['owner-id']); - return done(); - }); + makeMetadataRequest( + { + method: 'GET', + authCredentials: metadataAuthCredentials, + path: `/_/metadata/default/bucket/${bucket1}/${keyName}`, + }, + (err, res) => { + assert.ifError(err); + assert(res.body); + assert.strictEqual(res.statusCode, 200); + const body = JSON.parse(res.body); + assert(body['owner-id']); + return done(); + }, + ); }); it('should get an error for accessing invalid routes', done => { - makeMetadataRequest({ - method: 'GET', - authCredentials: metadataAuthCredentials, - path: '/_/metadata/admin/raft_sessions', - }, err => { - assert.strictEqual(err.code, 'NotImplemented'); - return done(); - }); + makeMetadataRequest( + { + method: 'GET', + authCredentials: metadataAuthCredentials, + path: '/_/metadata/admin/raft_sessions', + }, + err => { + assert.strictEqual(err.code, 'NotImplemented'); + return done(); + }, + ); }); }); diff --git a/tests/functional/raw-node/test/trailingChecksums.js b/tests/functional/raw-node/test/trailingChecksums.js index bad429c3c8..4c436c4781 100644 --- a/tests/functional/raw-node/test/trailingChecksums.js +++ b/tests/functional/raw-node/test/trailingChecksums.js @@ -7,9 +7,8 @@ const bucket = 'testunsupportedchecksumsbucket'; const objectKey = 'key'; const objData = Buffer.alloc(1024, 'a'); // note this is not the correct checksum in objDataWithTrailingChecksum -const objDataWithTrailingChecksum = '10\r\n0123456789abcdef\r\n' + - '10\r\n0123456789abcdef\r\n' + - '0\r\nx-amz-checksum-crc64nvme:YeIDuLa7tU0=\r\n'; +const objDataWithTrailingChecksum = + '10\r\n0123456789abcdef\r\n' + '10\r\n0123456789abcdef\r\n' + '0\r\nx-amz-checksum-crc64nvme:YeIDuLa7tU0=\r\n'; const objDataWithoutTrailingChecksum = '0123456789abcdef0123456789abcdef'; const config = require('../../config.json'); @@ -22,33 +21,47 @@ const itSkipIfAWS = process.env.AWS_ON_AIR ? it.skip : it; describe('trailing checksum requests:', () => { before(done => { - makeS3Request({ - method: 'PUT', - authCredentials, - bucket, - }, err => { - assert.ifError(err); - done(); - }); + makeS3Request( + { + method: 'PUT', + authCredentials, + bucket, + }, + err => { + assert.ifError(err); + done(); + }, + ); }); after(done => { - async.series([ - next => makeS3Request({ - method: 'DELETE', - authCredentials, - bucket, - objectKey, - }, next), - next => makeS3Request({ - method: 'DELETE', - authCredentials, - bucket, - }, next), - ], err => { - assert.ifError(err); - done(); - }); + async.series( + [ + next => + makeS3Request( + { + method: 'DELETE', + authCredentials, + bucket, + objectKey, + }, + next, + ), + next => + makeS3Request( + { + method: 'DELETE', + authCredentials, + bucket, + }, + next, + ), + ], + err => { + assert.ifError(err); + done(); + }, + ); }); it('should accept unsigned trailing checksum', done => { @@ -64,13 +77,13 @@ describe('trailing checksum requests:', () => { 'x-amz-trailer': 'x-amz-checksum-crc64nvme', }, }, - authCredentials + authCredentials, ), res => { assert.strictEqual(res.statusCode, 200); res.on('data', () => {}); res.on('end', done); - } + }, ); req.on('error', err => { @@ -85,18 +98,21 @@ describe('trailing checksum requests:', () => { }); it('should have correct object content for unsigned trailing checksum', done => { - makeS3Request({ - method: 'GET', - authCredentials, - bucket, - objectKey, - }, (err, res) => { - assert.ifError(err); - assert.strictEqual(res.statusCode, 200); - // check that the object data is the input stripped of the trailing checksum - assert.strictEqual(res.body, objDataWithoutTrailingChecksum); - return done(); - }); + makeS3Request( + { + method: 'GET', + authCredentials, + bucket, + objectKey, + }, + (err, res) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, 200); + // check that the object data is the input stripped of the trailing checksum + assert.strictEqual(res.body, objDataWithoutTrailingChecksum); + return done(); + }, + ); }); itSkipIfAWS('should respond with BadRequest for signed trailing checksum', done => { @@ -111,13 +127,13 @@ describe('trailing checksum requests:', () => { 'x-amz-trailer': 'x-amz-checksum-sha256', }, }, - authCredentials + authCredentials, ), res => { assert.strictEqual(res.statusCode, 400); res.on('data', () => {}); res.on('end', done); - } + }, ); req.on('error', err => { diff --git a/tests/functional/raw-node/test/unsignedChecksumHeaders.js b/tests/functional/raw-node/test/unsignedChecksumHeaders.js index 26340ff496..87b422891f 100644 --- a/tests/functional/raw-node/test/unsignedChecksumHeaders.js +++ b/tests/functional/raw-node/test/unsignedChecksumHeaders.js @@ -34,7 +34,7 @@ class HttpRequestAuthV4NoSHA256SignedHeader extends HttpRequestAuthV4 { const urlObj = new url.URL(this._url); const signedHeaders = { - 'host': urlObj.host, + host: urlObj.host, 'x-amz-date': this._timestamp, }; const httpHeaders = Object.assign({}, this._httpParams.headers); @@ -44,48 +44,63 @@ class HttpRequestAuthV4NoSHA256SignedHeader extends HttpRequestAuthV4 { signedHeaders[lowerHeader] = httpHeaders[header]; } }); - httpHeaders.Authorization = - this.getAuthorizationHeader(urlObj, signedHeaders, httpHeaders['x-amz-content-sha256']); + httpHeaders.Authorization = this.getAuthorizationHeader( + urlObj, + signedHeaders, + httpHeaders['x-amz-content-sha256'], + ); return Object.assign(httpHeaders, signedHeaders); } } describe('unsigned x-amz-content-sha256 header in AuthV4 requests:', () => { before(done => { - makeS3Request({ - method: 'PUT', - authCredentials, - bucket, - }, err => { - assert.ifError(err); - done(); - }); + makeS3Request( + { + method: 'PUT', + authCredentials, + bucket, + }, + err => { + assert.ifError(err); + done(); + }, + ); }); after(done => { - async.series([ - next => makeS3Request({ - method: 'DELETE', - authCredentials, - bucket, - objectKey, - }, next), - next => makeS3Request({ - method: 'DELETE', - authCredentials, - bucket, - }, next), - ], err => { - assert.ifError(err); - done(); - }); + async.series( + [ + next => + makeS3Request( + { + method: 'DELETE', + authCredentials, + bucket, + objectKey, + }, + next, + ), + next => + makeS3Request( + { + method: 'DELETE', + authCredentials, + bucket, + }, + next, + ), + ], + err => { + assert.ifError(err); + done(); + }, + ); }); it('should accept x-amz-content-sha256 header not in SignedHeaders list', done => { // Calculate the SHA256 hash of the data - const contentSha256 = crypto.createHash('sha256') - .update(objData) - .digest('hex'); + const contentSha256 = crypto.createHash('sha256').update(objData).digest('hex'); const req = new HttpRequestAuthV4NoSHA256SignedHeader( `http://localhost:8000/${bucket}/${objectKey}`, @@ -97,7 +112,7 @@ describe('unsigned x-amz-content-sha256 header in AuthV4 requests:', () => { 'x-amz-content-sha256': contentSha256, }, }, - authCredentials + authCredentials, ), res => { let body = ''; @@ -106,11 +121,14 @@ describe('unsigned x-amz-content-sha256 header in AuthV4 requests:', () => { }); res.on('end', () => { assert.strictEqual(body, '', 'expected empty body'); - assert.strictEqual(res.statusCode, 200, - 'Request should succeed even when x-amz-content-sha256 is not signed'); + assert.strictEqual( + res.statusCode, + 200, + 'Request should succeed even when x-amz-content-sha256 is not signed', + ); done(); }); - } + }, ); req.on('error', err => { diff --git a/tests/functional/raw-node/test/unsupportedChecksums.js b/tests/functional/raw-node/test/unsupportedChecksums.js index dd04be0f5f..ed3ac96ecf 100644 --- a/tests/functional/raw-node/test/unsupportedChecksums.js +++ b/tests/functional/raw-node/test/unsupportedChecksums.js @@ -15,25 +15,31 @@ const itSkipIfAWS = process.env.AWS_ON_AIR ? it.skip : it; describe('unsupported checksum requests:', () => { before(done => { - makeS3Request({ - method: 'PUT', - authCredentials, - bucket, - }, err => { - assert.ifError(err); - done(); - }); + makeS3Request( + { + method: 'PUT', + authCredentials, + bucket, + }, + err => { + assert.ifError(err); + done(); + }, + ); }); after(done => { - makeS3Request({ - method: 'DELETE', - authCredentials, - bucket, - }, err => { - assert.ifError(err); - done(); - }); + makeS3Request( + { + method: 'DELETE', + authCredentials, + bucket, + }, + err => { + assert.ifError(err); + done(); + }, + ); }); itSkipIfAWS('should respond with BadRequest for trailing checksum', done => { @@ -48,13 +54,13 @@ describe('unsupported checksum requests:', () => { 'x-amz-trailer': 'x-amz-checksum-sha256', }, }, - authCredentials + authCredentials, ), res => { assert.strictEqual(res.statusCode, 400); res.on('data', () => {}); res.on('end', done); - } + }, ); req.on('error', err => { diff --git a/tests/functional/raw-node/test/unsupportedQuries.js b/tests/functional/raw-node/test/unsupportedQuries.js index 9f3995f666..20e09f72ce 100644 --- a/tests/functional/raw-node/test/unsupportedQuries.js +++ b/tests/functional/raw-node/test/unsupportedQuries.js @@ -12,10 +12,8 @@ describe('unsupported query requests:', () => { const queryObj = {}; queryObj[query] = ''; - itSkipIfAWS(`should respond with NotImplemented for ?${query} request`, - done => { - makeS3Request({ method: 'GET', queryObj, bucket, objectKey }, - err => { + itSkipIfAWS(`should respond with NotImplemented for ?${query} request`, done => { + makeS3Request({ method: 'GET', queryObj, bucket, objectKey }, err => { assert.strictEqual(err.code, 'NotImplemented'); assert.strictEqual(err.statusCode, 501); done(); @@ -29,10 +27,8 @@ describe('unsupported bucket query requests:', () => { const queryObj = {}; queryObj[query] = ''; - itSkipIfAWS(`should respond with NotImplemented for ?${query} request`, - done => { - makeS3Request({ method: 'GET', queryObj, bucket }, - err => { + itSkipIfAWS(`should respond with NotImplemented for ?${query} request`, done => { + makeS3Request({ method: 'GET', queryObj, bucket }, err => { assert.strictEqual(err.code, 'NotImplemented'); assert.strictEqual(err.statusCode, 501); done(); diff --git a/tests/functional/raw-node/utils/HttpRequestAuthV4.js b/tests/functional/raw-node/utils/HttpRequestAuthV4.js index b979fd8ce2..0b83bad1e3 100644 --- a/tests/functional/raw-node/utils/HttpRequestAuthV4.js +++ b/tests/functional/raw-node/utils/HttpRequestAuthV4.js @@ -63,30 +63,24 @@ class HttpRequestAuthV4 extends stream.Writable { getCredentialScope() { const signingDate = this._timestamp.slice(0, 8); - const credentialScope = - `${signingDate}/${REGION}/${SERVICE}/aws4_request`; + const credentialScope = `${signingDate}/${REGION}/${SERVICE}/aws4_request`; // console.log(`CREDENTIAL SCOPE: "${credentialScope}"`); return credentialScope; } getSigningKey() { const signingDate = this._timestamp.slice(0, 8); - const dateKey = crypto.createHmac('sha256', `AWS4${this._secretKey}`) - .update(signingDate, 'binary').digest(); - const dateRegionKey = crypto.createHmac('sha256', dateKey) - .update(REGION, 'binary').digest(); - const dateRegionServiceKey = crypto.createHmac('sha256', dateRegionKey) - .update(SERVICE, 'binary').digest(); - this._signingKey = crypto.createHmac('sha256', dateRegionServiceKey) - .update('aws4_request', 'binary').digest(); + const dateKey = crypto.createHmac('sha256', `AWS4${this._secretKey}`).update(signingDate, 'binary').digest(); + const dateRegionKey = crypto.createHmac('sha256', dateKey).update(REGION, 'binary').digest(); + const dateRegionServiceKey = crypto.createHmac('sha256', dateRegionKey).update(SERVICE, 'binary').digest(); + this._signingKey = crypto.createHmac('sha256', dateRegionServiceKey).update('aws4_request', 'binary').digest(); } createSignature(stringToSign) { if (!this._signingKey) { this.getSigningKey(); } - return crypto.createHmac('sha256', this._signingKey) - .update(stringToSign).digest('hex'); + return crypto.createHmac('sha256', this._signingKey).update(stringToSign).digest('hex'); } getCanonicalRequest(urlObj, signedHeaders, contentSha256) { @@ -96,19 +90,16 @@ class HttpRequestAuthV4 extends stream.Writable { urlObj.searchParams.forEach((value, key) => { qsParams.push({ key, value }); }); - const canonicalQueryString = - qsParams - .sort((a, b) => { - if (a.key !== b.key) { - return a.key < b.key ? -1 : 1; - } - return a.value < b.value ? -1 : 1; - }) - .map(param => `${encodeURI(param.key)}=${encodeURI(param.value)}`) - .join('&'); - const canonicalSignedHeaders = signedHeadersList - .map(header => `${header}:${signedHeaders[header]}\n`) - .join(''); + const canonicalQueryString = qsParams + .sort((a, b) => { + if (a.key !== b.key) { + return a.key < b.key ? -1 : 1; + } + return a.value < b.value ? -1 : 1; + }) + .map(param => `${encodeURI(param.key)}=${encodeURI(param.value)}`) + .join('&'); + const canonicalSignedHeaders = signedHeadersList.map(header => `${header}:${signedHeaders[header]}\n`).join(''); const canonicalRequest = [ method, urlObj.pathname, @@ -123,41 +114,37 @@ class HttpRequestAuthV4 extends stream.Writable { } constructRequestStringToSign(canonicalReq) { - const canonicalReqHash = - crypto.createHash('sha256').update(canonicalReq).digest('hex'); - const stringToSign = `AWS4-HMAC-SHA256\n${this._timestamp}\n` + - `${this.getCredentialScope()}\n${canonicalReqHash}`; + const canonicalReqHash = crypto.createHash('sha256').update(canonicalReq).digest('hex'); + const stringToSign = + `AWS4-HMAC-SHA256\n${this._timestamp}\n` + `${this.getCredentialScope()}\n${canonicalReqHash}`; // console.log(`STRING TO SIGN: "${stringToSign}"`); return stringToSign; } getAuthorizationSignature(urlObj, signedHeaders, contentSha256) { - const canonicalRequest = - this.getCanonicalRequest(urlObj, signedHeaders, contentSha256); - this._lastSignature = this.createSignature( - this.constructRequestStringToSign(canonicalRequest)); + const canonicalRequest = this.getCanonicalRequest(urlObj, signedHeaders, contentSha256); + this._lastSignature = this.createSignature(this.constructRequestStringToSign(canonicalRequest)); return this._lastSignature; } getAuthorizationHeader(urlObj, signedHeaders, contentSha256) { - const authorizationSignature = - this.getAuthorizationSignature(urlObj, signedHeaders, contentSha256); + const authorizationSignature = this.getAuthorizationSignature(urlObj, signedHeaders, contentSha256); const signedHeadersList = Object.keys(signedHeaders).sort(); - return ['AWS4-HMAC-SHA256', - `Credential=${this._accessKey}/${this.getCredentialScope()},`, - `SignedHeaders=${signedHeadersList.join(';')},`, - `Signature=${authorizationSignature}`, - ].join(' '); + return [ + 'AWS4-HMAC-SHA256', + `Credential=${this._accessKey}/${this.getCredentialScope()},`, + `SignedHeaders=${signedHeadersList.join(';')},`, + `Signature=${authorizationSignature}`, + ].join(' '); } constructChunkStringToSign(chunkData) { - const currentChunkHash = - crypto.createHash('sha256').update(chunkData.toString()) - .digest('hex'); - const stringToSign = `AWS4-HMAC-SHA256-PAYLOAD\n${this._timestamp}\n` + - `${this.getCredentialScope()}\n${this._lastSignature}\n` + - `${EMPTY_STRING_HASH}\n${currentChunkHash}`; + const currentChunkHash = crypto.createHash('sha256').update(chunkData.toString()).digest('hex'); + const stringToSign = + `AWS4-HMAC-SHA256-PAYLOAD\n${this._timestamp}\n` + + `${this.getCredentialScope()}\n${this._lastSignature}\n` + + `${EMPTY_STRING_HASH}\n${currentChunkHash}`; // console.log(`CHUNK STRING TO SIGN: "${stringToSign}"`); return stringToSign; } @@ -173,13 +160,7 @@ class HttpRequestAuthV4 extends stream.Writable { return chunkData; } const chunkSignature = this.getChunkSignature(chunkData); - return [chunkData.length.toString(16), - ';chunk-signature=', - chunkSignature, - '\r\n', - chunkData, - '\r\n', - ].join(''); + return [chunkData.length.toString(16), ';chunk-signature=', chunkSignature, '\r\n', chunkData, '\r\n'].join(''); } _constructRequest(hasDataToSend) { @@ -196,7 +177,7 @@ class HttpRequestAuthV4 extends stream.Writable { const urlObj = new url.URL(this._url); const signedHeaders = { - 'host': urlObj.host, + host: urlObj.host, 'x-amz-date': this._timestamp, }; const httpHeaders = Object.assign({}, this._httpParams.headers); @@ -212,13 +193,11 @@ class HttpRequestAuthV4 extends stream.Writable { }); if (!signedHeaders['x-amz-content-sha256']) { if (hasDataToSend) { - signedHeaders['x-amz-content-sha256'] = - 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD'; + signedHeaders['x-amz-content-sha256'] = 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD'; signedHeaders['content-encoding'] = 'aws-chunked'; this._chunkedUpload = true; if (contentLengthHeader !== undefined) { - signedHeaders['x-amz-decoded-content-length'] = - httpHeaders[contentLengthHeader]; + signedHeaders['x-amz-decoded-content-length'] = httpHeaders[contentLengthHeader]; delete signedHeaders['content-length']; delete httpHeaders[contentLengthHeader]; httpHeaders['transfer-encoding'] = 'chunked'; @@ -227,8 +206,11 @@ class HttpRequestAuthV4 extends stream.Writable { signedHeaders['x-amz-content-sha256'] = EMPTY_STRING_HASH; } } - httpHeaders.Authorization = - this.getAuthorizationHeader(urlObj, signedHeaders, signedHeaders['x-amz-content-sha256']); + httpHeaders.Authorization = this.getAuthorizationHeader( + urlObj, + signedHeaders, + signedHeaders['x-amz-content-sha256'], + ); return Object.assign(httpHeaders, signedHeaders); } diff --git a/tests/functional/raw-node/utils/MetadataMock.js b/tests/functional/raw-node/utils/MetadataMock.js index 008a284b41..6b0a661b80 100644 --- a/tests/functional/raw-node/utils/MetadataMock.js +++ b/tests/functional/raw-node/utils/MetadataMock.js @@ -8,10 +8,10 @@ const dummyBucketMD = { WRITE: [], WRITE_ACP: [], READ: [], - READ_ACP: [] }, + READ_ACP: [], + }, _name: 'xxxfriday10', - _owner: - '94224c921648ada653f584f3caf42654ccf3f1cbd2e569a24e88eb460f2f84d8', + _owner: '94224c921648ada653f584f3caf42654ccf3f1cbd2e569a24e88eb460f2f84d8', _ownerDisplayName: 'test_1518720219', _creationDate: '2018-02-16T21:55:16.415Z', _mdBucketModelVersion: 5, @@ -33,10 +33,10 @@ const dummyBucketMD = { WRITE: [], WRITE_ACP: [], READ: [], - READ_ACP: [] }, + READ_ACP: [], + }, _name: 'xxxfriday11', - _owner: - '94224c921648ada653f584f3caf42654ccf3f1cbd2e569a24e88eb460f2f84d8', + _owner: '94224c921648ada653f584f3caf42654ccf3f1cbd2e569a24e88eb460f2f84d8', _ownerDisplayName: 'test_1518720219', _creationDate: '2018-02-16T21:55:16.415Z', _mdBucketModelVersion: 5, @@ -55,157 +55,207 @@ const dummyBucketMD = { const objectList = { Contents: [ - { key: 'testobject1', - value: JSON.stringify({ - 'owner-display-name': 'test_1518720219', - 'owner-id': - '94224c921648ada653f584f3caf42654ccf3f1cbd2e569a24e88eb460f2f84d8', - 'content-length': 0, - 'content-md5': 'd41d8cd98f00b204e9800998ecf8427e', - 'x-amz-version-id': 'null', - 'x-amz-server-version-id': '', - 'x-amz-storage-class': 'STANDARD', - 'x-amz-server-side-encryption': '', - 'x-amz-server-side-encryption-aws-kms-key-id': '', - 'x-amz-server-side-encryption-customer-algorithm': '', - 'x-amz-website-redirect-location': '', - 'acl': { - Canned: 'private', - FULL_CONTROL: [], - WRITE_ACP: [], - READ: [], - READ_ACP: [], - }, - 'key': '', - 'location': null, - 'isDeleteMarker': false, - 'tags': {}, - 'replicationInfo': { - status: '', - backends: [], - content: [], - destination: '', - storageClass: '', - role: '', - storageType: '', - dataStoreVersionId: '', - }, - 'dataStoreName': 'us-east-1', - 'last-modified': '2018-02-16T22:43:37.174Z', - 'md-model-version': 3, - }) }, + { + key: 'testobject1', + value: JSON.stringify({ + 'owner-display-name': 'test_1518720219', + 'owner-id': '94224c921648ada653f584f3caf42654ccf3f1cbd2e569a24e88eb460f2f84d8', + 'content-length': 0, + 'content-md5': 'd41d8cd98f00b204e9800998ecf8427e', + 'x-amz-version-id': 'null', + 'x-amz-server-version-id': '', + 'x-amz-storage-class': 'STANDARD', + 'x-amz-server-side-encryption': '', + 'x-amz-server-side-encryption-aws-kms-key-id': '', + 'x-amz-server-side-encryption-customer-algorithm': '', + 'x-amz-website-redirect-location': '', + acl: { + Canned: 'private', + FULL_CONTROL: [], + WRITE_ACP: [], + READ: [], + READ_ACP: [], + }, + key: '', + location: null, + isDeleteMarker: false, + tags: {}, + replicationInfo: { + status: '', + backends: [], + content: [], + destination: '', + storageClass: '', + role: '', + storageType: '', + dataStoreVersionId: '', + }, + dataStoreName: 'us-east-1', + 'last-modified': '2018-02-16T22:43:37.174Z', + 'md-model-version': 3, + }), + }, ], }; const mockLogs = { info: { start: 1, cseq: 7, prune: 1 }, log: [ - { db: 'friday', method: 0, entries: [ - { value: '{\"attributes\":\"{\\\"name\\\":\\\"friday\\\",' + - '\\\"owner\\\":\\\"94224c921648ada653f584f3caf42654ccf3f1cb' + - 'd2e569a24e88eb460f2f84d8\\\",\\\"ownerDisplayName\\\":' + - '\\\"test_1518720219\\\",\\\"creationDate\\\":' + - '\\\"2018-02-16T19:59:31.664Z\\\",\\\"mdBucketModelVersion\\\":5,' + - '\\\"transient\\\":true,\\\"deleted\\\":false,' + - '\\\"serverSideEncryption\\\":null,\\\"versioningConfiguration' + - '\\\":null,\\\"locationConstraint\\\":\\\"us-east-1\\\",\\\"cors' + - '\\\":null,\\\"replicationConfiguration\\\":null}\"}' }, - ] }, - { db: 'friday', method: 7, entries: [ - { value: '{\"attributes\":\"{\\\"name\\\":\\\"friday\\\",' + - '\\\"owner\\\":\\\"94224c921648ada653f584f3caf42654ccf3f1cb' + - 'd2e569a24e88eb460f2f84d8\\\",\\\"ownerDisplayName\\\":' + - '\\\"test_1518720219\\\",\\\"creationDate\\\":' + - '\\\"2018-02-16T19:59:31.664Z\\\",\\\"mdBucketModelVersion\\\":5,' + - '\\\"transient\\\":false,\\\"deleted\\\":false,' + - '\\\"serverSideEncryption\\\":null,\\\"versioningConfiguration' + - '\\\":null,\\\"locationConstraint\\\":\\\"us-east-1\\\",\\\"cors' + - '\\\":null,\\\"replicationConfiguration\\\":null}\",' + - '\"raftSession\":1}' }, - ] }, - { db: 'friday7', method: 0, entries: [ - { value: '{\"attributes\":\"{\\\"name\\\":\\\"friday7\\\",' + - '\\\"owner\\\":\\\"94224c921648ada653f584f3caf42654ccf3f1cb' + - 'd2e569a24e88eb460f2f84d8\\\",\\\"ownerDisplayName\\\":' + - '\\\"test_1518720219\\\",\\\"creationDate\\\":' + - '\\\"2018-02-16T20:41:34.253Z\\\",\\\"mdBucketModelVersion\\\":5,' + - '\\\"transient\\\":true,\\\"deleted\\\":false,' + - '\\\"serverSideEncryption\\\":null,\\\"versioningConfiguration' + - '\\\":null,\\\"locationConstraint\\\":\\\"us-east-1\\\",\\\"cors' + - '\\\":null,\\\"replicationConfiguration\\\":null}\"}' }, - ] }, - { db: 'friday7', method: 7, entries: [ - { value: '{\"attributes\":\"{\\\"name\\\":\\\"friday7\\\",' + - '\\\"owner\\\":\\\"94224c921648ada653f584f3caf42654ccf3f1cb' + - 'd2e569a24e88eb460f2f84d8\\\",\\\"ownerDisplayName\\\":' + - '\\\"test_1518720219\\\",\\\"creationDate\\\":' + - '\\\"2018-02-16T20:41:34.253Z\\\",\\\"mdBucketModelVersion\\\":5,' + - '\\\"transient\\\":false,\\\"deleted\\\":false,' + - '\\\"serverSideEncryption\\\":null,\\\"versioningConfiguration' + - '\\\":null,\\\"locationConstraint\\\":\\\"us-east-1\\\",\\\"cors' + - '\\\":null,\\\"replicationConfiguration\\\":null}\",' + - '\"raftSession\":1}' }, - ] }, - { db: 'xxxfriday10', method: 0, entries: [ - { value: '{\"attributes\":\"{\\\"name\\\":\\\"xxxfriday10\\\",' + - '\\\"owner\\\":\\\"94224c921648ada653f584f3caf42654ccf3f1cb' + - 'd2e569a24e88eb460f2f84d8\\\",\\\"ownerDisplayName\\\":' + - '\\\"test_1518720219\\\",\\\"creationDate\\\":' + - '\\\"2018-02-16T21:55:16.415Z\\\",\\\"mdBucketModelVersion\\\":5,' + - '\\\"transient\\\":true,\\\"deleted\\\":false,' + - '\\\"serverSideEncryption\\\":null,\\\"versioningConfiguration' + - '\\\":null,\\\"locationConstraint\\\":\\\"us-east-1\\\",\\\"cors' + - '\\\":null,\\\"replicationConfiguration\\\":null}\"}' }, - ] }, - { db: 'xxxfriday10', method: 7, entries: [ - { value: '{\"attributes\":\"{\\\"name\\\":\\\"xxxfriday10\\\",' + - '\\\"owner\\\":\\\"94224c921648ada653f584f3caf42654ccf3f1cb' + - 'd2e569a24e88eb460f2f84d8\\\",\\\"ownerDisplayName\\\":' + - '\\\"test_1518720219\\\",\\\"creationDate\\\":' + - '\\\"2018-02-16T21:55:16.415Z\\\",\\\"mdBucketModelVersion\\\":5,' + - '\\\"transient\\\":false,\\\"deleted\\\":false,' + - '\\\"serverSideEncryption\\\":null,\\\"versioningConfiguration' + - '\\\":null,\\\"locationConstraint\\\":\\\"us-east-1\\\",\\\"cors' + - '\\\":null,\\\"replicationConfiguration\\\":null}\",' + - '\"raftSession\":1}' }, - ] }, - { db: 'xxxfriday10', method: 8, entries: [ - { - key: 'afternoon', - value: '{\"owner-display-name\":\"test_1518720219\",' + - '\"owner-id\":\"94224c921648ada653f584f3caf42654ccf3f1cb' + - 'd2e569a24e88eb460f2f84d8\",\"content-length\":0,' + - '\"content-md5\":\"d41d8cd98f00b204e9800998ecf8427e\",' + - '\"x-amz-version-id\":\"null\",' + - '\"x-amz-server-version-id\":\"\",\"x-amz-storage-class' + - '\":\"STANDARD\",\"x-amz-server-side-encryption\":\"\",' + - '\"x-amz-server-side-encryption-aws-kms-key-id\":\"\",' + - '\"x-amz-server-side-encryption-customer-algorithm\":' + - '\"\",\"x-amz-website-redirect-location\":\"\",\"acl\":' + - '{\"Canned\":\"private\",\"FULL_CONTROL\":[],' + - '\"WRITE_ACP\":[],\"READ\":[],\"READ_ACP\":[]},\"key\":' + - '\"\",\"location\":null,\"isDeleteMarker\":false,\"tags' + - '\":{},\"replicationInfo\":{\"status\":\"\",\"backends\":' + - '[],\"content\":[],\"destination\":\"\",\"storageClass\":' + - '\"\",\"role\":\"\",\"storageType\":\"\",' + - '\"dataStoreVersionId\":\"\"},\"dataStoreName\":' + - '\"us-east-1\",\"last-modified\":\"2018-02-16T21:56:52.' + - '690Z\",\"md-model-version\":3}', - }, - ] }, - ] }; + { + db: 'friday', + method: 0, + entries: [ + { + value: + '{\"attributes\":\"{\\\"name\\\":\\\"friday\\\",' + + '\\\"owner\\\":\\\"94224c921648ada653f584f3caf42654ccf3f1cb' + + 'd2e569a24e88eb460f2f84d8\\\",\\\"ownerDisplayName\\\":' + + '\\\"test_1518720219\\\",\\\"creationDate\\\":' + + '\\\"2018-02-16T19:59:31.664Z\\\",\\\"mdBucketModelVersion\\\":5,' + + '\\\"transient\\\":true,\\\"deleted\\\":false,' + + '\\\"serverSideEncryption\\\":null,\\\"versioningConfiguration' + + '\\\":null,\\\"locationConstraint\\\":\\\"us-east-1\\\",\\\"cors' + + '\\\":null,\\\"replicationConfiguration\\\":null}\"}', + }, + ], + }, + { + db: 'friday', + method: 7, + entries: [ + { + value: + '{\"attributes\":\"{\\\"name\\\":\\\"friday\\\",' + + '\\\"owner\\\":\\\"94224c921648ada653f584f3caf42654ccf3f1cb' + + 'd2e569a24e88eb460f2f84d8\\\",\\\"ownerDisplayName\\\":' + + '\\\"test_1518720219\\\",\\\"creationDate\\\":' + + '\\\"2018-02-16T19:59:31.664Z\\\",\\\"mdBucketModelVersion\\\":5,' + + '\\\"transient\\\":false,\\\"deleted\\\":false,' + + '\\\"serverSideEncryption\\\":null,\\\"versioningConfiguration' + + '\\\":null,\\\"locationConstraint\\\":\\\"us-east-1\\\",\\\"cors' + + '\\\":null,\\\"replicationConfiguration\\\":null}\",' + + '\"raftSession\":1}', + }, + ], + }, + { + db: 'friday7', + method: 0, + entries: [ + { + value: + '{\"attributes\":\"{\\\"name\\\":\\\"friday7\\\",' + + '\\\"owner\\\":\\\"94224c921648ada653f584f3caf42654ccf3f1cb' + + 'd2e569a24e88eb460f2f84d8\\\",\\\"ownerDisplayName\\\":' + + '\\\"test_1518720219\\\",\\\"creationDate\\\":' + + '\\\"2018-02-16T20:41:34.253Z\\\",\\\"mdBucketModelVersion\\\":5,' + + '\\\"transient\\\":true,\\\"deleted\\\":false,' + + '\\\"serverSideEncryption\\\":null,\\\"versioningConfiguration' + + '\\\":null,\\\"locationConstraint\\\":\\\"us-east-1\\\",\\\"cors' + + '\\\":null,\\\"replicationConfiguration\\\":null}\"}', + }, + ], + }, + { + db: 'friday7', + method: 7, + entries: [ + { + value: + '{\"attributes\":\"{\\\"name\\\":\\\"friday7\\\",' + + '\\\"owner\\\":\\\"94224c921648ada653f584f3caf42654ccf3f1cb' + + 'd2e569a24e88eb460f2f84d8\\\",\\\"ownerDisplayName\\\":' + + '\\\"test_1518720219\\\",\\\"creationDate\\\":' + + '\\\"2018-02-16T20:41:34.253Z\\\",\\\"mdBucketModelVersion\\\":5,' + + '\\\"transient\\\":false,\\\"deleted\\\":false,' + + '\\\"serverSideEncryption\\\":null,\\\"versioningConfiguration' + + '\\\":null,\\\"locationConstraint\\\":\\\"us-east-1\\\",\\\"cors' + + '\\\":null,\\\"replicationConfiguration\\\":null}\",' + + '\"raftSession\":1}', + }, + ], + }, + { + db: 'xxxfriday10', + method: 0, + entries: [ + { + value: + '{\"attributes\":\"{\\\"name\\\":\\\"xxxfriday10\\\",' + + '\\\"owner\\\":\\\"94224c921648ada653f584f3caf42654ccf3f1cb' + + 'd2e569a24e88eb460f2f84d8\\\",\\\"ownerDisplayName\\\":' + + '\\\"test_1518720219\\\",\\\"creationDate\\\":' + + '\\\"2018-02-16T21:55:16.415Z\\\",\\\"mdBucketModelVersion\\\":5,' + + '\\\"transient\\\":true,\\\"deleted\\\":false,' + + '\\\"serverSideEncryption\\\":null,\\\"versioningConfiguration' + + '\\\":null,\\\"locationConstraint\\\":\\\"us-east-1\\\",\\\"cors' + + '\\\":null,\\\"replicationConfiguration\\\":null}\"}', + }, + ], + }, + { + db: 'xxxfriday10', + method: 7, + entries: [ + { + value: + '{\"attributes\":\"{\\\"name\\\":\\\"xxxfriday10\\\",' + + '\\\"owner\\\":\\\"94224c921648ada653f584f3caf42654ccf3f1cb' + + 'd2e569a24e88eb460f2f84d8\\\",\\\"ownerDisplayName\\\":' + + '\\\"test_1518720219\\\",\\\"creationDate\\\":' + + '\\\"2018-02-16T21:55:16.415Z\\\",\\\"mdBucketModelVersion\\\":5,' + + '\\\"transient\\\":false,\\\"deleted\\\":false,' + + '\\\"serverSideEncryption\\\":null,\\\"versioningConfiguration' + + '\\\":null,\\\"locationConstraint\\\":\\\"us-east-1\\\",\\\"cors' + + '\\\":null,\\\"replicationConfiguration\\\":null}\",' + + '\"raftSession\":1}', + }, + ], + }, + { + db: 'xxxfriday10', + method: 8, + entries: [ + { + key: 'afternoon', + value: + '{\"owner-display-name\":\"test_1518720219\",' + + '\"owner-id\":\"94224c921648ada653f584f3caf42654ccf3f1cb' + + 'd2e569a24e88eb460f2f84d8\",\"content-length\":0,' + + '\"content-md5\":\"d41d8cd98f00b204e9800998ecf8427e\",' + + '\"x-amz-version-id\":\"null\",' + + '\"x-amz-server-version-id\":\"\",\"x-amz-storage-class' + + '\":\"STANDARD\",\"x-amz-server-side-encryption\":\"\",' + + '\"x-amz-server-side-encryption-aws-kms-key-id\":\"\",' + + '\"x-amz-server-side-encryption-customer-algorithm\":' + + '\"\",\"x-amz-website-redirect-location\":\"\",\"acl\":' + + '{\"Canned\":\"private\",\"FULL_CONTROL\":[],' + + '\"WRITE_ACP\":[],\"READ\":[],\"READ_ACP\":[]},\"key\":' + + '\"\",\"location\":null,\"isDeleteMarker\":false,\"tags' + + '\":{},\"replicationInfo\":{\"status\":\"\",\"backends\":' + + '[],\"content\":[],\"destination\":\"\",\"storageClass\":' + + '\"\",\"role\":\"\",\"storageType\":\"\",' + + '\"dataStoreVersionId\":\"\"},\"dataStoreName\":' + + '\"us-east-1\",\"last-modified\":\"2018-02-16T21:56:52.' + + '690Z\",\"md-model-version\":3}', + }, + ], + }, + ], +}; -const mockLogString = '\\/_\\/raft_sessions\\/[\\d]*\\/log\\?begin=' + - '[\\d]*&limit=[\\d]*&targetLeader=false'; +const mockLogString = '\\/_\\/raft_sessions\\/[\\d]*\\/log\\?begin=' + '[\\d]*&limit=[\\d]*&targetLeader=false'; const mockLogURLRegex = new RegExp(mockLogString); class MetadataMock { onRequest(req, res) { if (req.method !== 'GET') { res.writeHead(501); - return res.end(JSON.stringify({ - error: 'mock server only supports GET requests', - })); + return res.end( + JSON.stringify({ + error: 'mock server only supports GET requests', + }), + ); } if (/\/_\/raft_sessions\/[1-8]\/bucket/.test(req.url)) { const value = ['bucket1', 'bucket2', 'users..bucket']; @@ -214,30 +264,41 @@ class MetadataMock { } else if (/\/default\/attributes\/[a-z0-9]/.test(req.url)) { const bucketName = req.url.split('/'); const bucketMd = dummyBucketMD[bucketName[bucketName.length - 1]]; - const dummyBucketMdObj = new BucketInfo(bucketMd._name, - bucketMd._owner, bucketMd._ownerDisplayName, - bucketMd._creationDate, bucketMd._mdBucketModelVersion, - bucketMd._acl, bucketMd._transient, bucketMd._deleted, + const dummyBucketMdObj = new BucketInfo( + bucketMd._name, + bucketMd._owner, + bucketMd._ownerDisplayName, + bucketMd._creationDate, + bucketMd._mdBucketModelVersion, + bucketMd._acl, + bucketMd._transient, + bucketMd._deleted, bucketMd._serverSideEncryption, - bucketMd.versioningConfiguration, bucketMd._locationContraint, - bucketMd._websiteConfiguration, bucketMd._cors, - bucketMd._lifeCycle); + bucketMd.versioningConfiguration, + bucketMd._locationContraint, + bucketMd._websiteConfiguration, + bucketMd._cors, + bucketMd._lifeCycle, + ); return res.end(dummyBucketMdObj.serialize()); - } else if - (/\/default\/bucket\/.*?listingType=Delimiter/.test(req.url)) { + } else if (/\/default\/bucket\/.*?listingType=Delimiter/.test(req.url)) { return res.end(JSON.stringify(objectList)); } else if (/\/default\/bucket\/.*\/.*?/.test(req.url)) { - return res.end(JSON.stringify({ - 'owner-id': '123', - 'metadata': 'dogsAreGood', - })); + return res.end( + JSON.stringify({ + 'owner-id': '123', + metadata: 'dogsAreGood', + }), + ); } else if (mockLogURLRegex.test(req.url)) { return res.end(JSON.stringify(mockLogs)); } res.writeHead(404); - return res.end(JSON.stringify({ - error: 'invalid path', - })); + return res.end( + JSON.stringify({ + error: 'invalid path', + }), + ); } } diff --git a/tests/functional/raw-node/utils/gcpUtils.js b/tests/functional/raw-node/utils/gcpUtils.js index 773fd0ec83..ea6e1d40a7 100644 --- a/tests/functional/raw-node/utils/gcpUtils.js +++ b/tests/functional/raw-node/utils/gcpUtils.js @@ -11,8 +11,7 @@ const genUniqID = () => { const genBucketName = testName => `cldsrvci-${testName}-${genUniqID()}`; -const defaultShouldRetry = err => - err && (err.name === 'SlowDown' || err.$metadata?.httpStatusCode === 429); +const defaultShouldRetry = err => err && (err.name === 'SlowDown' || err.$metadata?.httpStatusCode === 429); async function gcpRetryCall(callFn, retryOptions) { const { @@ -33,9 +32,9 @@ async function gcpRetryCall(callFn, retryOptions) { } const delay = getDelayMs(attempt); process.stdout.write( - 'Retryable error from GCP, retrying in ' + - `${delay}ms (attempt ${attempt + 1}): ${err}\n`); - + 'Retryable error from GCP, retrying in ' + `${delay}ms (attempt ${attempt + 1}): ${err}\n`, + ); + await new Promise(resolve => setTimeout(resolve, delay)); } } @@ -45,41 +44,40 @@ async function gcpRetryCall(callFn, retryOptions) { async function gcpRetry(gcpClient, command, retryOptions, cb) { if (cb) { - return callbackify(() => gcpRetry(gcpClient, command, - retryOptions))(cb); + return callbackify(() => gcpRetry(gcpClient, command, retryOptions))(cb); } return gcpRetryCall(() => gcpClient.send(command), retryOptions); } -const defaultShouldRetryUpload = err => err && ( - err.name === 'NoSuchBucket' - || err.name === 'NotFound' - || err.$metadata?.httpStatusCode === 404 - || err.name === 'SlowDown' - || err.$metadata?.httpStatusCode === 429 - || (typeof err.message === 'string' - && (err.message.includes('NoSuchBucket') - || err.message.includes('unable to complete upload'))) -); - -const defaultShouldRetryMpuCreate = err => err && ( - err.name === 'NoSuchBucket' - || err.name === 'NotFound' - || err.$metadata?.httpStatusCode === 404 - || err.name === 'SlowDown' - || err.$metadata?.httpStatusCode === 429 -); +const defaultShouldRetryUpload = err => + err && + (err.name === 'NoSuchBucket' || + err.name === 'NotFound' || + err.$metadata?.httpStatusCode === 404 || + err.name === 'SlowDown' || + err.$metadata?.httpStatusCode === 429 || + (typeof err.message === 'string' && + (err.message.includes('NoSuchBucket') || err.message.includes('unable to complete upload')))); + +const defaultShouldRetryMpuCreate = err => + err && + (err.name === 'NoSuchBucket' || + err.name === 'NotFound' || + err.$metadata?.httpStatusCode === 404 || + err.name === 'SlowDown' || + err.$metadata?.httpStatusCode === 429); async function gcpUploadWithRetry(gcpClient, params, retryOptions) { - const callFn = () => new Promise((resolve, reject) => { - gcpClient.upload(params, (err, data) => { - if (err) { - return reject(err); - } - return resolve(data); + const callFn = () => + new Promise((resolve, reject) => { + gcpClient.upload(params, (err, data) => { + if (err) { + return reject(err); + } + return resolve(data); + }); }); - }); return gcpRetryCall(callFn, { maxAttempts: 6, @@ -90,10 +88,10 @@ async function gcpUploadWithRetry(gcpClient, params, retryOptions) { } async function gcpCreateMultipartUploadWithRetry(gcpClient, params, retryOptions) { - const callFn = () => new Promise((resolve, reject) => { - gcpClient.createMultipartUpload(params, - (err, res) => (err ? reject(err) : resolve(res))); - }); + const callFn = () => + new Promise((resolve, reject) => { + gcpClient.createMultipartUpload(params, (err, res) => (err ? reject(err) : resolve(res))); + }); return gcpRetryCall(callFn, { maxAttempts: 6, shouldRetry: defaultShouldRetryMpuCreate, @@ -106,52 +104,63 @@ async function gcpCreateMultipartUploadWithRetry(gcpClient, params, retryOptions function gcpMpuSetup(params, callback) { const { gcpClient, bucketNames, key, partCount, partSize } = params; - return async.waterfall([ - next => gcpCreateMultipartUploadWithRetry(gcpClient, { - Bucket: bucketNames.mpu.Name, - Key: key, - }) - .then(res => next(null, res.UploadId)) - .catch(err => next(err)), - (uploadId, next) => { - if (partCount <= 0) { - return next('SkipPutPart', { uploadId }); - } - const arrayData = Array.from(Array(partCount).keys()); - const etagList = Array(partCount); - let count = 0; - return async.eachLimit(arrayData, 10, - (info, moveOn) => { - gcpClient.uploadPart({ + return async.waterfall( + [ + next => + gcpCreateMultipartUploadWithRetry(gcpClient, { Bucket: bucketNames.mpu.Name, Key: key, - UploadId: uploadId, - PartNumber: info + 1, - Body: Buffer.alloc(partSize), - ContentLength: partSize, - }, (err, res) => { - if (err) { - return moveOn(err); - } - if (!(++count % 100)) { - process.stdout.write(`Uploaded Parts: ${count}\n`); - } - etagList[info] = res.ETag; - return moveOn(null); - }); - }, err => { - next(err, { uploadId, etagList }); - }); - }, - ], (err, result) => { - if (err) { - if (err === 'SkipPutPart') { - return callback(null, result); + }) + .then(res => next(null, res.UploadId)) + .catch(err => next(err)), + (uploadId, next) => { + if (partCount <= 0) { + return next('SkipPutPart', { uploadId }); + } + const arrayData = Array.from(Array(partCount).keys()); + const etagList = Array(partCount); + let count = 0; + return async.eachLimit( + arrayData, + 10, + (info, moveOn) => { + gcpClient.uploadPart( + { + Bucket: bucketNames.mpu.Name, + Key: key, + UploadId: uploadId, + PartNumber: info + 1, + Body: Buffer.alloc(partSize), + ContentLength: partSize, + }, + (err, res) => { + if (err) { + return moveOn(err); + } + if (!(++count % 100)) { + process.stdout.write(`Uploaded Parts: ${count}\n`); + } + etagList[info] = res.ETag; + return moveOn(null); + }, + ); + }, + err => { + next(err, { uploadId, etagList }); + }, + ); + }, + ], + (err, result) => { + if (err) { + if (err === 'SkipPutPart') { + return callback(null, result); + } + return callback(err); } - return callback(err); - } - return callback(null, result); - }); + return callback(null, result); + }, + ); } function genPutTagObj(size, duplicate) { @@ -200,12 +209,13 @@ function genDelTagObj(size, tagPrefix) { const regionalLoc = 'us-west1'; const multiRegionalLoc = 'us'; function setBucketClass(storageClass) { - const locationConstraint = - storageClass === 'REGIONAL' ? regionalLoc : multiRegionalLoc; - return '' + + const locationConstraint = storageClass === 'REGIONAL' ? regionalLoc : multiRegionalLoc; + return ( + '' + `${locationConstraint}` + `${storageClass}` + - ''; + '' + ); } async function waitForBucketReady(gcpClient, bucketName, retryOptions) { diff --git a/tests/functional/raw-node/utils/makeRequest.js b/tests/functional/raw-node/utils/makeRequest.js index 6759160c2e..20e50e37a9 100644 --- a/tests/functional/raw-node/utils/makeRequest.js +++ b/tests/functional/raw-node/utils/makeRequest.js @@ -11,18 +11,18 @@ const constructStringToSignV2 = require('arsenal/build/lib/auth/v2/constructStri function signGcpRequest(request, credentials, date) { if (!credentials || !credentials.secretKey || !credentials.accessKey) { - throw new Error('Invalid GCP credentials: must have accessKey and secretKey properties. ' + - `Got: ${JSON.stringify(credentials)}`); - } + throw new Error( + 'Invalid GCP credentials: must have accessKey and secretKey properties. ' + + `Got: ${JSON.stringify(credentials)}`, + ); + } // eslint-disable-next-line no-param-reassign request.headers['x-goog-date'] = date.toUTCString(); const data = Object.assign({}, request.headers); const logger = { trace: () => {} }; const stringToSign = constructStringToSignV2(request, data, logger, 'GCP'); // Sign with HMAC-SHA1 - const signature = crypto.createHmac('sha1', credentials.secretKey) - .update(stringToSign) - .digest('base64'); + const signature = crypto.createHmac('sha1', credentials.secretKey).update(stringToSign).digest('base64'); // eslint-disable-next-line no-param-reassign request.headers['Authorization'] = `GOOG1 ${credentials.accessKey}:${signature}`; } @@ -73,9 +73,18 @@ function _decodeURI(uri) { * @return {undefined} - and call callback */ function makeRequest(params, callback) { - const { hostname, port, method, queryObj, headers, path, - authCredentials, requestBody, jsonResponse, - urlForSignature } = params; + const { + hostname, + port, + method, + queryObj, + headers, + path, + authCredentials, + requestBody, + jsonResponse, + urlForSignature, + } = params; const options = { hostname, port, @@ -139,8 +148,16 @@ function makeRequest(params, callback) { // decode path because signing code re-encodes it req.path = _decodeURI(encodedPath); if (authCredentials && !params.GCP) { - auth.client.generateV4Headers(req, queryObj || '', - authCredentials.accessKey, authCredentials.secretKey, 's3', undefined, undefined, requestBody); + auth.client.generateV4Headers( + req, + queryObj || '', + authCredentials.accessKey, + authCredentials.secretKey, + 's3', + undefined, + undefined, + requestBody, + ); } // restore original URL-encoded path req.path = savedPath; @@ -167,8 +184,7 @@ function makeRequest(params, callback) { * @return {undefined} - and call callback */ function makeS3Request(params, callback) { - const { method, queryObj, headers, bucket, objectKey, authCredentials, requestBody } - = params; + const { method, queryObj, headers, bucket, objectKey, authCredentials, requestBody } = params; const options = { authCredentials, hostname: process.env.AWS_ON_AIR ? 's3.amazonaws.com' : ipAddress, @@ -202,8 +218,7 @@ function makeS3Request(params, callback) { * @return {undefined} - and call callback */ function makeBackbeatRequest(params, callback) { - const { method, headers, bucket, objectKey, resourceType, - authCredentials, requestBody, queryObj } = params; + const { method, headers, bucket, objectKey, resourceType, authCredentials, requestBody, queryObj } = params; const options = { authCredentials, hostname: ipAddress, diff --git a/tests/functional/report/master.json b/tests/functional/report/master.json index cc83b902dd..9fe2fc66b9 100644 --- a/tests/functional/report/master.json +++ b/tests/functional/report/master.json @@ -1,6 +1,6 @@ { "tests": { - "files": [ "/test" ], + "files": ["/test"], "on": "aggressor" } } diff --git a/tests/functional/report/monitoring.js b/tests/functional/report/monitoring.js index 529aeaebef..4a5bcf4ca1 100644 --- a/tests/functional/report/monitoring.js +++ b/tests/functional/report/monitoring.js @@ -7,13 +7,20 @@ describe('Monitoring - getting metrics', () => { const conf = require('../config.json'); async function query(path, method = 'GET', token = 'report-token-1') { - return new Promise(resolve => http.request({ - method, - host: conf.ipAddress, - path, - port: 8000, - headers: { 'x-scal-report-token': token }, - }, () => resolve()).end()); + return new Promise(resolve => + http + .request( + { + method, + host: conf.ipAddress, + path, + port: 8000, + headers: { 'x-scal-report-token': token }, + }, + () => resolve(), + ) + .end(), + ); } async function getMetrics() { @@ -22,14 +29,18 @@ describe('Monitoring - getting metrics', () => { assert.strictEqual(res.statusCode, 200); const body = []; - res.on('data', chunk => { body.push(chunk); }); + res.on('data', chunk => { + body.push(chunk); + }); res.on('end', () => resolve(body.join(''))); }); }); } function parseMetric(metrics, name, labels) { - const labelsString = Object.entries(labels).map(e => `${e[0]}="${e[1]}"`).join(','); + const labelsString = Object.entries(labels) + .map(e => `${e[0]}="${e[1]}"`) + .join(','); const metric = metrics.match(new RegExp(`^${name}{${labelsString}} (.*)$`, 'm')); return metric ? metric[1] : null; } @@ -51,26 +62,26 @@ describe('Monitoring - getting metrics', () => { [ // Check all methods are reported (on unsupported route) - ['/_/fooooo', { method: 'GET', code: '400' }], - ['/_/fooooo', { method: 'PUT', code: '400' }], - ['/_/fooooo', { method: 'POST', code: '400' }], - ['/_/fooooo', { method: 'DELETE', code: '400' }], + ['/_/fooooo', { method: 'GET', code: '400' }], + ['/_/fooooo', { method: 'PUT', code: '400' }], + ['/_/fooooo', { method: 'POST', code: '400' }], + ['/_/fooooo', { method: 'DELETE', code: '400' }], // S3/api routes - ['/', { method: 'GET', code: '403', action: 'serviceGet' }], - ['/foo', { method: 'GET', code: '404', action: 'bucketGet' }], - ['/foo/bar', { method: 'GET', code: '404', action: 'objectGet' }], + ['/', { method: 'GET', code: '403', action: 'serviceGet' }], + ['/foo', { method: 'GET', code: '404', action: 'bucketGet' }], + ['/foo/bar', { method: 'GET', code: '404', action: 'objectGet' }], // Internal handlers - ['/_/report', { method: 'GET', code: '200', action: 'report' }], - ['/_/backbeat', { method: 'GET', code: '405', action: 'routeBackbeat' }], - ['/_/metadata', { method: 'GET', code: '403', action: 'routeMetadata' }], - ['/_/workflow-engine-operator', - { method: 'GET', code: '405', action: 'routeWorkflowEngineOperator' }], + ['/_/report', { method: 'GET', code: '200', action: 'report' }], + ['/_/backbeat', { method: 'GET', code: '405', action: 'routeBackbeat' }], + ['/_/metadata', { method: 'GET', code: '403', action: 'routeMetadata' }], + ['/_/workflow-engine-operator', { method: 'GET', code: '405', action: 'routeWorkflowEngineOperator' }], ].forEach(([path, labels]) => { it(`should count http ${labels.method} requests metrics on ${path}`, async () => { const count = parseRequestsCount(await getMetrics(), labels); - for (let i = 1; i <= 3; i++) { /* eslint no-await-in-loop: "off" */ + for (let i = 1; i <= 3; i++) { + /* eslint no-await-in-loop: "off" */ await query(path, labels.method); const c = parseRequestsCount(await getMetrics(), labels); diff --git a/tests/functional/s3cmd/tests.js b/tests/functional/s3cmd/tests.js index 9140714ad6..9d05cfceb3 100644 --- a/tests/functional/s3cmd/tests.js +++ b/tests/functional/s3cmd/tests.js @@ -14,11 +14,7 @@ const emptyUpload = 'Utest0B'; const emptyDownload = 'Dtest0B'; const download = 'tmpfile'; const MPUpload = 'test60MB'; -const MPUploadSplitter = [ - 'test60..|..MB', - '..|..test60MB', - 'test60MB..|..', -]; +const MPUploadSplitter = ['test60..|..MB', '..|..test60MB', 'test60MB..|..']; const MPDownload = 'MPtmpfile'; const MPDownloadCopy = 'MPtmpfile2'; const downloadCopy = 'tmpfile2'; @@ -51,8 +47,7 @@ function diff(putFile, receivedFile, done) { function createFile(name, bytes, callback) { process.stdout.write(`dd if=/dev/urandom of=${name} bs=${bytes} count=1\n`); - const ret = proc.spawnSync('dd', ['if=/dev/urandom', `of=${name}`, - `bs=${bytes}`, 'count=1'], { stdio: 'inherit' }); + const ret = proc.spawnSync('dd', ['if=/dev/urandom', `of=${name}`, `bs=${bytes}`, 'count=1'], { stdio: 'inherit' }); assert.strictEqual(ret.status, 0); callback(); } @@ -81,8 +76,7 @@ function exec(args, done, exitCode) { } process.stdout.write(`${program} ${av}\n`); const ret = proc.spawnSync(program, av, { stdio: 'inherit' }); - assert.strictEqual(ret.status, exit, - 's3cmd did not yield expected exit status.'); + assert.strictEqual(ret.status, exit, 's3cmd did not yield expected exit status.'); done(); } @@ -106,12 +100,16 @@ function checkRawOutput(args, lineFinder, testString, stream, cb) { }); child.on('close', () => { if (stream === 'stderr') { - const foundIt = allErrData.join('').split('\n') + const foundIt = allErrData + .join('') + .split('\n') .filter(item => item.indexOf(lineFinder) > -1) .some(item => item.indexOf(testString) > -1); return cb(foundIt); } - const foundIt = allData.join('').split('\n') + const foundIt = allData + .join('') + .split('\n') .filter(item => item.indexOf(lineFinder) > -1) .some(item => item.indexOf(testString) > -1); return cb(foundIt); @@ -123,12 +121,12 @@ function findEndString(data, start) { const end = data.length; for (let i = start + 1; i < end; ++i) { if (data[i] === delimiter) { - return (i); + return i; } else if (data[i] === '\\') { ++i; } } - return (-1); + return -1; } function findEndJson(data, start) { @@ -143,10 +141,10 @@ function findEndJson(data, start) { i = findEndString(data, i); } if (count === 0) { - return (i); + return i; } } - return (-1); + return -1; } function readJsonFromChild(child, lineFinder, cb) { @@ -160,9 +158,11 @@ function readJsonFromChild(child, lineFinder, cb) { const findLine = data.indexOf(lineFinder); const findBrace = data.indexOf('{', findLine); const findEnd = findEndJson(data, findBrace); - const endJson = data.substring(findBrace, findEnd + 1) - .replace(/"/g, '\\"').replace(/'/g, '"') - .replace(/b'/g, '\'') + const endJson = data + .substring(findBrace, findEnd + 1) + .replace(/"/g, '\\"') + .replace(/'/g, '"') + .replace(/b'/g, "'") .replace(/b"/g, '"'); return cb(JSON.parse(endJson)); }); @@ -213,37 +213,32 @@ function retrieveInfo() { function createEncryptedBucket(name, cb) { const res = retrieveInfo(); const prog = `${__dirname}/../../../bin/create_encrypted_bucket.js`; - let args = [ - prog, - '-a', res.accessKey, - '-k', res.secretKey, - '-b', name, - '-h', res.host, - '-p', res.port, - '-v', - ]; + let args = [prog, '-a', res.accessKey, '-k', res.secretKey, '-b', name, '-h', res.host, '-p', res.port, '-v']; if (conf.https) { args = args.concat('-s'); } const body = []; - const child = proc.spawn(args[0], args) - .on('exit', () => { - const hasSucceed = body.join('').split('\n').find(item => { - const json = safeJSONParse(item); - const test = !(json instanceof Error) && json.name === 'S3' && - json.statusCode === 200; - if (test) { - return true; + const child = proc + .spawn(args[0], args) + .on('exit', () => { + const hasSucceed = body + .join('') + .split('\n') + .find(item => { + const json = safeJSONParse(item); + const test = !(json instanceof Error) && json.name === 'S3' && json.statusCode === 200; + if (test) { + return true; + } + return false; + }); + if (!hasSucceed) { + process.stderr.write(`${body.join('')}\n`); + return cb(new Error('Cannot create encrypted bucket')); } - return false; - }); - if (!hasSucceed) { - process.stderr.write(`${body.join('')}\n`); - return cb(new Error('Cannot create encrypted bucket')); - } - return cb(); - }) - .on('error', cb); + return cb(); + }) + .on('error', cb); child.stdout.on('data', chunk => body.push(chunk.toString())); } @@ -257,10 +252,7 @@ describe('s3cmd putBucket', () => { // pass by returning error. If legacyAWSBehvior, request // would return a 200 it('put the same bucket, should fail', done => { - exec([ - 'mb', `s3://${bucket}`, - '--bucket-location=scality-us-west-1', - ], done, 13); + exec(['mb', `s3://${bucket}`, '--bucket-location=scality-us-west-1'], done, 13); }); it('put an invalid bucket, should fail', done => { @@ -276,16 +268,15 @@ describe('s3cmd putBucket', () => { }); if (process.env.ENABLE_KMS_ENCRYPTION === 'true') { - it('creates a valid bucket with server side encryption', - function f(done) { - this.timeout(5000); - exec(['rb', `s3://${bucket}`], err => { - if (err) { - return done(err); - } - return createEncryptedBucket(bucket, done); - }); - }); + it('creates a valid bucket with server side encryption', function f(done) { + this.timeout(5000); + exec(['rb', `s3://${bucket}`], err => { + if (err) { + return done(err); + } + return createEncryptedBucket(bucket, done); + }); + }); } }); @@ -299,23 +290,18 @@ describe('s3cmd put and get bucket ACLs', function aclBuck() { }); it('should get canned ACL that was set', done => { - checkRawOutput(['info', `s3://${bucket}`], 'ACL', '*anon*: READ', - 'stdout', foundIt => { + checkRawOutput(['info', `s3://${bucket}`], 'ACL', '*anon*: READ', 'stdout', foundIt => { assert(foundIt); done(); }); }); it('should set a specific ACL', done => { - exec([ - 'setacl', `s3://${bucket}`, - `--acl-grant=write:${emailAccount}`, - ], done); + exec(['setacl', `s3://${bucket}`, `--acl-grant=write:${emailAccount}`], done); }); it('should get specific ACL that was set', done => { - checkRawOutput(['info', `s3://${bucket}`], 'ACL', - `${lowerCaseEmail}: WRITE`, 'stdout', foundIt => { + checkRawOutput(['info', `s3://${bucket}`], 'ACL', `${lowerCaseEmail}: WRITE`, 'stdout', foundIt => { assert(foundIt); done(); }); @@ -340,23 +326,18 @@ describe('s3cmd getService', () => { }); }); - it("should have response headers matching AWS's response headers", - done => { - provideLineOfInterest(['ls', '--debug'], '\'headers\': {', - parsedObject => { - assert(parsedObject['x-amz-id-2']); - assert(parsedObject['transfer-encoding']); - assert(parsedObject['x-amz-request-id']); - const gmtDate = new Date(parsedObject.date) - .toUTCString(); - assert.strictEqual(parsedObject.date, gmtDate); - assert.strictEqual(parsedObject - ['content-type'], 'application/xml'); - assert.strictEqual(parsedObject - ['set-cookie'], undefined); - done(); - }); + it("should have response headers matching AWS's response headers", done => { + provideLineOfInterest(['ls', '--debug'], "'headers': {", parsedObject => { + assert(parsedObject['x-amz-id-2']); + assert(parsedObject['transfer-encoding']); + assert(parsedObject['x-amz-request-id']); + const gmtDate = new Date(parsedObject.date).toUTCString(); + assert.strictEqual(parsedObject.date, gmtDate); + assert.strictEqual(parsedObject['content-type'], 'application/xml'); + assert.strictEqual(parsedObject['set-cookie'], undefined); + done(); }); + }); }); describe('s3cmd putObject', function toto() { @@ -409,10 +390,7 @@ describe('s3cmd copyObject without MPU to same bucket', function copyStuff() { }); it('should copy an object to the same bucket', done => { - exec([ - 'cp', `s3://${bucket}/${upload}`, - `s3://${bucket}/${upload}copy`, - ], done); + exec(['cp', `s3://${bucket}/${upload}`, `s3://${bucket}/${upload}copy`], done); }); it('should get an object that was copied', done => { @@ -428,42 +406,36 @@ describe('s3cmd copyObject without MPU to same bucket', function copyStuff() { }); }); -describe('s3cmd copyObject without MPU to different bucket ' + - '(always unencrypted)', - function copyStuff() { - const copyBucket = 'receiverbucket'; - this.timeout(40000); +describe('s3cmd copyObject without MPU to different bucket ' + '(always unencrypted)', function copyStuff() { + const copyBucket = 'receiverbucket'; + this.timeout(40000); - before('create receiver bucket', done => { - exec(['mb', `s3://${copyBucket}`], done); - }); + before('create receiver bucket', done => { + exec(['mb', `s3://${copyBucket}`], done); + }); - after('delete downloaded file and receiver bucket' + - 'copied', done => { - deleteFile(downloadCopy, () => { - exec(['rb', `s3://${copyBucket}`], done); - }); + after('delete downloaded file and receiver bucket' + 'copied', done => { + deleteFile(downloadCopy, () => { + exec(['rb', `s3://${copyBucket}`], done); }); + }); - it('should copy an object to the new bucket', done => { - exec([ - 'cp', `s3://${bucket}/${upload}`, - `s3://${copyBucket}/${upload}`, - ], done); - }); + it('should copy an object to the new bucket', done => { + exec(['cp', `s3://${bucket}/${upload}`, `s3://${copyBucket}/${upload}`], done); + }); - it('should get an object that was copied', done => { - exec(['get', `s3://${copyBucket}/${upload}`, downloadCopy], done); - }); + it('should get an object that was copied', done => { + exec(['get', `s3://${copyBucket}/${upload}`, downloadCopy], done); + }); - it('downloaded copy file should equal original uploaded file', done => { - diff(upload, downloadCopy, done); - }); + it('downloaded copy file should equal original uploaded file', done => { + diff(upload, downloadCopy, done); + }); - it('should delete copy of object', done => { - exec(['rm', `s3://${copyBucket}/${upload}`], done); - }); + it('should delete copy of object', done => { + exec(['rm', `s3://${copyBucket}/${upload}`], done); }); +}); describe('s3cmd put and get object ACLs', function aclObj() { this.timeout(60000); @@ -475,30 +447,25 @@ describe('s3cmd put and get object ACLs', function aclObj() { }); it('should get canned ACL that was set', done => { - checkRawOutput(['info', `s3://${bucket}/${upload}`], 'ACL', - '*anon*: READ', 'stdout', foundIt => { + checkRawOutput(['info', `s3://${bucket}/${upload}`], 'ACL', '*anon*: READ', 'stdout', foundIt => { assert(foundIt); done(); }); }); it('should set a specific ACL', done => { - exec(['setacl', `s3://${bucket}/${upload}`, - `--acl-grant=read:${emailAccount}`], done); + exec(['setacl', `s3://${bucket}/${upload}`, `--acl-grant=read:${emailAccount}`], done); }); it('should get specific ACL that was set', done => { - checkRawOutput(['info', `s3://${bucket}/${upload}`], 'ACL', - `${lowerCaseEmail}: READ`, 'stdout', foundIt => { + checkRawOutput(['info', `s3://${bucket}/${upload}`], 'ACL', `${lowerCaseEmail}: READ`, 'stdout', foundIt => { assert(foundIt); done(); }); }); - it('should return error if set acl for ' + - 'nonexistent object', done => { - exec(['setacl', `s3://${bucket}/${nonexist}`, - '--acl-public'], done, 12); + it('should return error if set acl for ' + 'nonexistent object', done => { + exec(['setacl', `s3://${bucket}/${nonexist}`, '--acl-public'], done, 12); }); }); @@ -508,16 +475,14 @@ describe('s3cmd delObject', () => { }); it('delete an already deleted object, should return a 204', done => { - provideLineOfInterest(['rm', `s3://${bucket}/${upload}`, '--debug'], - 'DEBUG: Response:\n{', parsedObject => { + provideLineOfInterest(['rm', `s3://${bucket}/${upload}`, '--debug'], 'DEBUG: Response:\n{', parsedObject => { assert.strictEqual(parsedObject.status, 204); done(); }); }); it('delete non-existing object, should return a 204', done => { - provideLineOfInterest(['rm', `s3://${bucket}/${nonexist}`, '--debug'], - 'DEBUG: Response:\n{', parsedObject => { + provideLineOfInterest(['rm', `s3://${bucket}/${nonexist}`, '--debug'], 'DEBUG: Response:\n{', parsedObject => { assert.strictEqual(parsedObject.status, 204); done(); }); @@ -596,10 +561,7 @@ describe('s3cmd multipart upload', function titi() { }); it('should copy an object that was put via multipart upload', done => { - exec([ - 'cp', `s3://${bucket}/${MPUpload}`, - `s3://${bucket}/${MPUpload}copy`, - ], done); + exec(['cp', `s3://${bucket}/${MPUpload}`, `s3://${bucket}/${MPUpload}copy`], done); }); it('should get an object that was copied', done => { @@ -663,9 +625,7 @@ MPUploadSplitter.forEach(file => { }); }); - -describe('s3cmd put, get and delete object with spaces ' + - 'in object key names', function test() { +describe('s3cmd put, get and delete object with spaces ' + 'in object key names', function test() { this.timeout(0); const keyWithSpacesAndPluses = 'key with spaces and + pluses +'; before('create file to put', done => { @@ -688,13 +648,11 @@ describe('s3cmd put, get and delete object with spaces ' + }); it('should get file with spaces', done => { - exec(['get', `s3://${bucket}/${keyWithSpacesAndPluses}`, download], - done); + exec(['get', `s3://${bucket}/${keyWithSpacesAndPluses}`, download], done); }); it('should list bucket showing file with spaces', done => { - checkRawOutput(['ls', `s3://${bucket}`], `s3://${bucket}`, - keyWithSpacesAndPluses, 'stdout', foundIt => { + checkRawOutput(['ls', `s3://${bucket}`], `s3://${bucket}`, keyWithSpacesAndPluses, 'stdout', foundIt => { assert(foundIt); done(); }); @@ -726,27 +684,26 @@ describe('s3cmd info', () => { // test that POLICY and CORS are returned as 'none' it('should find that policy has a value of none', done => { - checkRawOutput(['info', `s3://${bucket}`], 'Policy', 'none', - 'stdout', foundIt => { + checkRawOutput(['info', `s3://${bucket}`], 'Policy', 'none', 'stdout', foundIt => { assert(foundIt); done(); }); }); it('should find that cors has a value of none', done => { - checkRawOutput(['info', `s3://${bucket}`], 'CORS', 'none', - 'stdout', foundIt => { + checkRawOutput(['info', `s3://${bucket}`], 'CORS', 'none', 'stdout', foundIt => { assert(foundIt); done(); }); }); describe('after putting cors configuration', () => { - const corsConfig = '' + - 'PUT' + - 'http://www.allowedorigin.com' + - ''; + const corsConfig = + '' + + 'PUT' + + 'http://www.allowedorigin.com' + + ''; const filename = 'corss3cmdfile'; beforeEach(done => { @@ -760,8 +717,7 @@ describe('s3cmd info', () => { }); it('should find that cors has a value', done => { - checkRawOutput(['info', `s3://${bucket}`], 'CORS', corsConfig, - 'stdout', foundIt => { + checkRawOutput(['info', `s3://${bucket}`], 'CORS', corsConfig, 'stdout', foundIt => { assert(foundIt, 'Did not find value for cors'); done(); }); @@ -793,12 +749,14 @@ describe('s3cmd recursive delete with objects put by MPU', () => { this.timeout(120000); exec(['mb', `s3://${bucket}`], () => { createFile(upload16MB, 16777216, () => { - async.timesLimit(50, 1, (n, next) => { - exec([ - 'put', upload16MB, `s3://${bucket}/key${n}`, - '--multipart-chunk-size-mb=5', - ], next); - }, done); + async.timesLimit( + 50, + 1, + (n, next) => { + exec(['put', upload16MB, `s3://${bucket}/key${n}`, '--multipart-chunk-size-mb=5'], next); + }, + done, + ); }); }); }); @@ -822,9 +780,7 @@ describeSkipIfE2E('If no location is sent with the request', () => { // WARNING: change "us-east-1" to another locationConstraint depending // on the restEndpoints (./config.json) it('endpoint should be used to determine the locationConstraint', done => { - checkRawOutput(['info', `s3://${bucket}`], 'Location', 'us-east-1', - 'stdout', - foundIt => { + checkRawOutput(['info', `s3://${bucket}`], 'Location', 'us-east-1', 'stdout', foundIt => { assert(foundIt); done(); }); diff --git a/tests/functional/s3curl/tests.js b/tests/functional/s3curl/tests.js index c6562eff97..3d3b858875 100644 --- a/tests/functional/s3curl/tests.js +++ b/tests/functional/s3curl/tests.js @@ -22,8 +22,7 @@ const aclBucket = 'acluniverse'; const nonexist = 'nonexist'; const prefix = 'topLevel'; const delimiter = '/'; -let ownerCanonicalId = '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d52' - + '18e7cd47ef2be'; +let ownerCanonicalId = '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d52' + '18e7cd47ef2be'; const endpoint = `${transport}://${ipAddress}:8000`; // Let's precompute a few paths @@ -51,11 +50,9 @@ function diff(putFile, receivedFile, done) { }); } - function createFile(name, bytes, callback) { process.stdout.write(`dd if=/dev/urandom of=${name} bs=${bytes} count=1\n`); - let ret = proc.spawnSync('dd', ['if=/dev/urandom', `of=${name}`, - `bs=${bytes}`, 'count=1'], { stdio: 'inherit' }); + let ret = proc.spawnSync('dd', ['if=/dev/urandom', `of=${name}`, `bs=${bytes}`, 'count=1'], { stdio: 'inherit' }); assert.strictEqual(ret.status, 0); process.stdout.write(`chmod ugoa+rw ${name}\n`); ret = proc.spawnSync('chmod', ['ugo+rw', name], { stdio: 'inherit' }); @@ -101,15 +98,13 @@ function provideRawOutput(args, cb) { httpCode = lines.find(line => { const trimmed = line.trim().toUpperCase(); // ignore 100 Continue HTTP code - if (trimmed.startsWith('HTTP/1.1 ') && - !trimmed.includes('100 CONTINUE')) { + if (trimmed.startsWith('HTTP/1.1 ') && !trimmed.includes('100 CONTINUE')) { return true; } return false; }); if (httpCode) { - httpCode = httpCode.trim().replace('HTTP/1.1 ', '') - .toUpperCase(); + httpCode = httpCode.trim().replace('HTTP/1.1 ', '').toUpperCase(); } else { process.stdout.write(`${lines.join('\n')}\n`); return cb(new Error("Can't find line in http response code")); @@ -134,15 +129,13 @@ function provideRawOutput(args, cb) { * @return {undefined} */ function putObjects(filepath, objectPaths, cb) { - provideRawOutput( - [`--put=${filepath}`, '--', objectPaths[0], '-v'], - httpCode => { - assert.strictEqual(httpCode, '200 OK'); - if (objectPaths.length > 1) { - return putObjects(filepath, objectPaths.slice(1), cb); - } - return cb(); - }); + provideRawOutput([`--put=${filepath}`, '--', objectPaths[0], '-v'], httpCode => { + assert.strictEqual(httpCode, '200 OK'); + if (objectPaths.length > 1) { + return putObjects(filepath, objectPaths.slice(1), cb); + } + return cb(); + }); } /** @@ -157,15 +150,13 @@ function putObjects(filepath, objectPaths, cb) { * @return {undefined} */ function deleteRemoteItems(items, cb) { - provideRawOutput( - ['--delete', '--', items[0], '-v'], - httpCode => { - assert.strictEqual(httpCode, '204 NO CONTENT'); - if (items.length > 1) { - return deleteRemoteItems(items.slice(1), cb); - } - return cb(); - }); + provideRawOutput(['--delete', '--', items[0], '-v'], httpCode => { + assert.strictEqual(httpCode, '204 NO CONTENT'); + if (items.length > 1) { + return deleteRemoteItems(items.slice(1), cb); + } + return cb(); + }); } describe('s3curl put delete buckets', () => { @@ -175,84 +166,68 @@ describe('s3curl put delete buckets', () => { }); it('should put a valid bucket', done => { - provideRawOutput( - ['--createBucket', '--', bucketPath, '-v'], - httpCode => { - assert.strictEqual(httpCode, '200 OK'); - done(); - }); + provideRawOutput(['--createBucket', '--', bucketPath, '-v'], httpCode => { + assert.strictEqual(httpCode, '200 OK'); + done(); + }); }); - it('should return 409 error in new regions and 200 in us-east-1 ' + - '(legacyAWSBehvior) when try to put a bucket with a name ' + - 'already being used', done => { - provideRawOutput(['--createBucket', '--', bucketPath, '-v'], - httpCode => { - assert(httpCode === '200 OK' - || httpCode === '409 CONFLICT'); + it( + 'should return 409 error in new regions and 200 in us-east-1 ' + + '(legacyAWSBehvior) when try to put a bucket with a name ' + + 'already being used', + done => { + provideRawOutput(['--createBucket', '--', bucketPath, '-v'], httpCode => { + assert(httpCode === '200 OK' || httpCode === '409 CONFLICT'); done(); }); - }); + }, + ); - it('should not be able to put a bucket with invalid xml' + - ' in the post body', done => { - provideRawOutput([ - '--createBucket', - '--', - '--data', - 'malformedxml', - bucketPath, - '-v', - ], (httpCode, rawOutput) => { - assert.strictEqual(httpCode, '400 BAD REQUEST'); - assertError(rawOutput.stdout, 'MalformedXML', - done); - }); + it('should not be able to put a bucket with invalid xml' + ' in the post body', done => { + provideRawOutput( + ['--createBucket', '--', '--data', 'malformedxml', bucketPath, '-v'], + (httpCode, rawOutput) => { + assert.strictEqual(httpCode, '400 BAD REQUEST'); + assertError(rawOutput.stdout, 'MalformedXML', done); + }, + ); }); - it('should not be able to put a bucket with xml that does' + - ' not conform to s3 docs for locationConstraint', done => { - provideRawOutput([ - '--createBucket', - '--', - '--data', - 'a', - bucketPath, - '-v', - ], (httpCode, rawOutput) => { - assert.strictEqual(httpCode, '400 BAD REQUEST'); - assertError(rawOutput.stdout, 'MalformedXML', - done); - }); - }); + it( + 'should not be able to put a bucket with xml that does' + ' not conform to s3 docs for locationConstraint', + done => { + provideRawOutput( + ['--createBucket', '--', '--data', 'a', bucketPath, '-v'], + (httpCode, rawOutput) => { + assert.strictEqual(httpCode, '400 BAD REQUEST'); + assertError(rawOutput.stdout, 'MalformedXML', done); + }, + ); + }, + ); it('should not be able to put a bucket with an invalid name', done => { - provideRawOutput( - ['--createBucket', '--', `${endpoint}/2`, '-v'], - (httpCode, rawOutput) => { - assert.strictEqual(httpCode, '400 BAD REQUEST'); - assertError(rawOutput.stdout, 'InvalidBucketName', done); - }); + provideRawOutput(['--createBucket', '--', `${endpoint}/2`, '-v'], (httpCode, rawOutput) => { + assert.strictEqual(httpCode, '400 BAD REQUEST'); + assertError(rawOutput.stdout, 'InvalidBucketName', done); + }); }); it('should not be able to put a bucket with an empty name', done => { - provideRawOutput( - ['--createBucket', '--', `${endpoint}/`, '-v'], - httpCode => { - assert.strictEqual(httpCode, '405 METHOD NOT ALLOWED'); - done(); - }); + provideRawOutput(['--createBucket', '--', `${endpoint}/`, '-v'], httpCode => { + assert.strictEqual(httpCode, '405 METHOD NOT ALLOWED'); + done(); + }); }); }); describe('s3curl delete bucket', () => { before(done => { - provideRawOutput( - ['--createBucket', '--', bucketPath, '-v'], - httpCode => { - assert.strictEqual(httpCode, '200 OK'); - done(); - }); + provideRawOutput(['--createBucket', '--', bucketPath, '-v'], httpCode => { + assert.strictEqual(httpCode, '200 OK'); + done(); + }); }); after(done => { @@ -264,316 +239,256 @@ describe('s3curl put delete buckets', () => { }); it('should not be able to get a bucket that was deleted', done => { - provideRawOutput( - ['--', bucketPath, '-v'], - (httpCode, rawOutput) => { - assert.strictEqual(httpCode, '404 NOT FOUND'); - assertError(rawOutput.stdout, 'NoSuchBucket', done); - }); + provideRawOutput(['--', bucketPath, '-v'], (httpCode, rawOutput) => { + assert.strictEqual(httpCode, '404 NOT FOUND'); + assertError(rawOutput.stdout, 'NoSuchBucket', done); + }); }); - it('should be able to create a bucket with a name' + - 'of a bucket that has previously been deleted', done => { - provideRawOutput( - ['--createBucket', '--', bucketPath, '-v'], - httpCode => { - assert.strictEqual(httpCode, '200 OK'); - done(); - }); + it('should be able to create a bucket with a name' + 'of a bucket that has previously been deleted', done => { + provideRawOutput(['--createBucket', '--', bucketPath, '-v'], httpCode => { + assert.strictEqual(httpCode, '200 OK'); + done(); + }); }); }); }); describe('s3curl put and get bucket ACLs', () => { after(done => { - deleteRemoteItems([ - `${endpoint}/${aclBucket}`, - `${endpoint}/${aclBucket}2`, - ], done); + deleteRemoteItems([`${endpoint}/${aclBucket}`, `${endpoint}/${aclBucket}2`], done); }); it('should be able to create a bucket with a canned ACL', done => { - provideRawOutput([ - '--createBucket', - '--', - '-H', - 'x-amz-acl:public-read', - `${endpoint}/${aclBucket}`, - '-v', - ], httpCode => { - assert.strictEqual(httpCode, '200 OK'); - done(); - }); - }); - - it('should be able to get a canned ACL', done => { provideRawOutput( - ['--', `${endpoint}/${aclBucket}?acl`, '-v'], - (httpCode, rawOutput) => { + ['--createBucket', '--', '-H', 'x-amz-acl:public-read', `${endpoint}/${aclBucket}`, '-v'], + httpCode => { assert.strictEqual(httpCode, '200 OK'); - parseString(rawOutput.stdout, (err, xml) => { - if (err) { - assert.ifError(err); - } - assert.strictEqual(xml.AccessControlPolicy - .Owner[0].ID[0], ownerCanonicalId); - assert.strictEqual(xml.AccessControlPolicy - .AccessControlList[0].Grant[0] - .Grantee[0].ID[0], ownerCanonicalId); - assert.strictEqual(xml.AccessControlPolicy - .AccessControlList[0].Grant[0] - .Permission[0], 'FULL_CONTROL'); - assert.strictEqual(xml.AccessControlPolicy - .AccessControlList[0].Grant[1] - .Grantee[0].URI[0], - 'http://acs.amazonaws.com/groups/global/AllUsers'); - assert.strictEqual(xml.AccessControlPolicy - .AccessControlList[0].Grant[1] - .Permission[0], 'READ'); - done(); - }); - }); + done(); + }, + ); }); - it('should be able to create a bucket with a specific ACL', done => { - provideRawOutput([ - '--createBucket', - '--', - '-H', - 'x-amz-grant-read:uri=' + - 'http://acs.amazonaws.com/groups/global/AllUsers', - `${endpoint}/${aclBucket}2`, - '-v', - ], httpCode => { + it('should be able to get a canned ACL', done => { + provideRawOutput(['--', `${endpoint}/${aclBucket}?acl`, '-v'], (httpCode, rawOutput) => { assert.strictEqual(httpCode, '200 OK'); - done(); + parseString(rawOutput.stdout, (err, xml) => { + if (err) { + assert.ifError(err); + } + assert.strictEqual(xml.AccessControlPolicy.Owner[0].ID[0], ownerCanonicalId); + assert.strictEqual( + xml.AccessControlPolicy.AccessControlList[0].Grant[0].Grantee[0].ID[0], + ownerCanonicalId, + ); + assert.strictEqual(xml.AccessControlPolicy.AccessControlList[0].Grant[0].Permission[0], 'FULL_CONTROL'); + assert.strictEqual( + xml.AccessControlPolicy.AccessControlList[0].Grant[1].Grantee[0].URI[0], + 'http://acs.amazonaws.com/groups/global/AllUsers', + ); + assert.strictEqual(xml.AccessControlPolicy.AccessControlList[0].Grant[1].Permission[0], 'READ'); + done(); + }); }); }); - it('should be able to get a specifically set ACL', done => { + it('should be able to create a bucket with a specific ACL', done => { provideRawOutput( - ['--', `${endpoint}/${aclBucket}2?acl`, '-v'], - (httpCode, rawOutput) => { + [ + '--createBucket', + '--', + '-H', + 'x-amz-grant-read:uri=' + 'http://acs.amazonaws.com/groups/global/AllUsers', + `${endpoint}/${aclBucket}2`, + '-v', + ], + httpCode => { assert.strictEqual(httpCode, '200 OK'); - parseString(rawOutput.stdout, (err, xml) => { - if (err) { - assert.ifError(err); - } - assert.strictEqual(xml.AccessControlPolicy - .Owner[0].ID[0], ownerCanonicalId); - assert.strictEqual(xml.AccessControlPolicy - .AccessControlList[0].Grant[0] - .Grantee[0].URI[0], - 'http://acs.amazonaws.com/groups/global/AllUsers'); - assert.strictEqual(xml.AccessControlPolicy - .AccessControlList[0].Grant[0] - .Permission[0], 'READ'); - done(); - }); + done(); + }, + ); + }); + + it('should be able to get a specifically set ACL', done => { + provideRawOutput(['--', `${endpoint}/${aclBucket}2?acl`, '-v'], (httpCode, rawOutput) => { + assert.strictEqual(httpCode, '200 OK'); + parseString(rawOutput.stdout, (err, xml) => { + if (err) { + assert.ifError(err); + } + assert.strictEqual(xml.AccessControlPolicy.Owner[0].ID[0], ownerCanonicalId); + assert.strictEqual( + xml.AccessControlPolicy.AccessControlList[0].Grant[0].Grantee[0].URI[0], + 'http://acs.amazonaws.com/groups/global/AllUsers', + ); + assert.strictEqual(xml.AccessControlPolicy.AccessControlList[0].Grant[0].Permission[0], 'READ'); + done(); }); + }); }); }); describe('s3curl getService', () => { before(done => { - provideRawOutput( - ['--createBucket', '--', bucketPath, '-v'], - httpCode => { + provideRawOutput(['--createBucket', '--', bucketPath, '-v'], httpCode => { + assert.strictEqual(httpCode, '200 OK'); + provideRawOutput(['--createBucket', '--', `${endpoint}/${aclBucket}`, '-v'], httpCode => { assert.strictEqual(httpCode, '200 OK'); - provideRawOutput( - ['--createBucket', '--', `${endpoint}/${aclBucket}`, '-v'], - httpCode => { - assert.strictEqual(httpCode, '200 OK'); - done(); - }); + done(); }); + }); }); after(done => { - deleteRemoteItems([ - bucketPath, - `${endpoint}/${aclBucket}`, - ], done); + deleteRemoteItems([bucketPath, `${endpoint}/${aclBucket}`], done); }); it('should get a list of all buckets created by user account', done => { - provideRawOutput( - ['--', `${endpoint}`, '-v'], - (httpCode, rawOutput) => { - assert.strictEqual(httpCode, '200 OK'); - parseString(rawOutput.stdout, (err, xml) => { - if (err) { - assert.ifError(err); - } - const bucketNames = xml.ListAllMyBucketsResult - .Buckets[0].Bucket - .map(item => item.Name[0]); - const whereIsMyBucket = bucketNames.indexOf(bucket); - assert(whereIsMyBucket > -1); - const whereIsMyAclBucket = bucketNames.indexOf(aclBucket); - assert(whereIsMyAclBucket > -1); - done(); - }); + provideRawOutput(['--', `${endpoint}`, '-v'], (httpCode, rawOutput) => { + assert.strictEqual(httpCode, '200 OK'); + parseString(rawOutput.stdout, (err, xml) => { + if (err) { + assert.ifError(err); + } + const bucketNames = xml.ListAllMyBucketsResult.Buckets[0].Bucket.map(item => item.Name[0]); + const whereIsMyBucket = bucketNames.indexOf(bucket); + assert(whereIsMyBucket > -1); + const whereIsMyAclBucket = bucketNames.indexOf(aclBucket); + assert(whereIsMyAclBucket > -1); + done(); }); + }); }); }); describe('s3curl putObject', () => { before(done => { - provideRawOutput( - ['--createBucket', '--', bucketPath, '-v'], - httpCode => { - assert.strictEqual(httpCode, '200 OK'); - createFile(upload, 1048576, done); - }); + provideRawOutput(['--createBucket', '--', bucketPath, '-v'], httpCode => { + assert.strictEqual(httpCode, '200 OK'); + createFile(upload, 1048576, done); + }); }); after(done => { - deleteRemoteItems([ - `${prefixedPath}${upload}1`, - `${prefixedPath}${upload}2`, - `${prefixedPath}${upload}3`, - bucketPath, - ], done); + deleteRemoteItems( + [`${prefixedPath}${upload}1`, `${prefixedPath}${upload}2`, `${prefixedPath}${upload}3`, bucketPath], + done, + ); }); // curl behavior is not consistent across the environments // skipping the test for now - it.skip('should not be able to put an object if request does not have ' + - 'content-length header', - done => { - provideRawOutput([ - '--debug', - `--put=${upload}`, - '--', - '-H', - 'content-length:', - `${prefixedPath}${upload}1`, - '-v', - ], (httpCode, rawOutput) => { + it.skip('should not be able to put an object if request does not have ' + 'content-length header', done => { + provideRawOutput( + ['--debug', `--put=${upload}`, '--', '-H', 'content-length:', `${prefixedPath}${upload}1`, '-v'], + (httpCode, rawOutput) => { assert.strictEqual(httpCode, '411 LENGTH REQUIRED'); assertError(rawOutput.stdout, 'MissingContentLength', done); - }); - }); - - it('should not be able to put an object if content-md5 header is ' + - 'invalid', - done => { - provideRawOutput(['--debug', `--put=${upload}`, - '--contentMd5', 'toto', '--', - `${endpoint}/${bucket}/` + - `${prefix}${delimiter}${upload}1`, '-v'], - (httpCode, rawOutput) => { - assert.strictEqual(httpCode, '400 BAD REQUEST'); - assertError(rawOutput.stdout, 'InvalidDigest', done); - }); - }); - - // skip until we figure out how to parse the response in the CI - it.skip('should not be able to put an object if content-md5 header is ' + - 'mismatched MD5', - done => { - provideRawOutput(['--debug', `--put=${upload}`, - '--contentMd5', 'rL0Y20zC+Fzt72VPzMSk2A==', '--', - `${endpoint}/${bucket}/` + - `${prefix}${delimiter}${upload}1`, '-v'], - (httpCode, rawOutput) => { - assert.strictEqual(httpCode, '400 BAD REQUEST'); - assertError(rawOutput.stdout, 'BadDigest', done); - }); - }); + }, + ); + }); - it('should not be able to put an object if using streaming ' + - 'chunked-upload with a valid V2 signature', - done => { - provideRawOutput([ + it('should not be able to put an object if content-md5 header is ' + 'invalid', done => { + provideRawOutput( + [ '--debug', `--put=${upload}`, + '--contentMd5', + 'toto', '--', - '-H', - 'x-amz-content-sha256: STREAMING-AWS4-HMAC-SHA256-PAYLOAD', - `${endpoint}/${bucket}/${prefix}${delimiter}${upload}1`, - '-v'], - (httpCode, rawOutput) => { - assert.strictEqual(httpCode, '400 BAD REQUEST'); - assertError(rawOutput.stdout, 'InvalidArgument', done); - }); - }); + `${endpoint}/${bucket}/` + `${prefix}${delimiter}${upload}1`, + '-v', + ], + (httpCode, rawOutput) => { + assert.strictEqual(httpCode, '400 BAD REQUEST'); + assertError(rawOutput.stdout, 'InvalidDigest', done); + }, + ); + }); - it('should not be able to put an object in a bucket with an invalid name', - done => { - provideRawOutput([ + // skip until we figure out how to parse the response in the CI + it.skip('should not be able to put an object if content-md5 header is ' + 'mismatched MD5', done => { + provideRawOutput( + [ '--debug', `--put=${upload}`, + '--contentMd5', + 'rL0Y20zC+Fzt72VPzMSk2A==', '--', - `${endpoint}/2/${basePath}${upload}1`, + `${endpoint}/${bucket}/` + `${prefix}${delimiter}${upload}1`, '-v', - ], (httpCode, rawOutput) => { + ], + (httpCode, rawOutput) => { assert.strictEqual(httpCode, '400 BAD REQUEST'); - assertError(rawOutput.stdout, 'InvalidBucketName', done); - }); - }); + assertError(rawOutput.stdout, 'BadDigest', done); + }, + ); + }); - it('should not be able to put an object in a bucket that does not exist', - done => { - provideRawOutput([ + it('should not be able to put an object if using streaming ' + 'chunked-upload with a valid V2 signature', done => { + provideRawOutput( + [ '--debug', `--put=${upload}`, '--', - `${endpoint}/${nonexist}/${basePath}${upload}1`, + '-H', + 'x-amz-content-sha256: STREAMING-AWS4-HMAC-SHA256-PAYLOAD', + `${endpoint}/${bucket}/${prefix}${delimiter}${upload}1`, '-v', - ], (httpCode, rawOutput) => { + ], + (httpCode, rawOutput) => { + assert.strictEqual(httpCode, '400 BAD REQUEST'); + assertError(rawOutput.stdout, 'InvalidArgument', done); + }, + ); + }); + + it('should not be able to put an object in a bucket with an invalid name', done => { + provideRawOutput( + ['--debug', `--put=${upload}`, '--', `${endpoint}/2/${basePath}${upload}1`, '-v'], + (httpCode, rawOutput) => { + assert.strictEqual(httpCode, '400 BAD REQUEST'); + assertError(rawOutput.stdout, 'InvalidBucketName', done); + }, + ); + }); + + it('should not be able to put an object in a bucket that does not exist', done => { + provideRawOutput( + ['--debug', `--put=${upload}`, '--', `${endpoint}/${nonexist}/${basePath}${upload}1`, '-v'], + (httpCode, rawOutput) => { assert.strictEqual(httpCode, '404 NOT FOUND'); assertError(rawOutput.stdout, 'NoSuchBucket', done); - }); - }); + }, + ); + }); - it('should not be able to put an object in a bucket with an empty name', - done => { - provideRawOutput([ - '--debug', - `--put=${upload}`, - '--', - `${endpoint}//${basePath}/${upload}1`, - '-v', - ], httpCode => { - assert.strictEqual(httpCode, '405 METHOD NOT ALLOWED'); - done(); - }); + it('should not be able to put an object in a bucket with an empty name', done => { + provideRawOutput( + ['--debug', `--put=${upload}`, '--', `${endpoint}//${basePath}/${upload}1`, '-v'], + httpCode => { + assert.strictEqual(httpCode, '405 METHOD NOT ALLOWED'); + done(); + }, + ); }); - it('should put first object in existing bucket with prefix ' + - 'and delimiter', done => { - provideRawOutput([ - '--debug', - `--put=${upload}`, - '--', - `${prefixedPath}${upload}1`, - '-v', - ], httpCode => { + it('should put first object in existing bucket with prefix ' + 'and delimiter', done => { + provideRawOutput(['--debug', `--put=${upload}`, '--', `${prefixedPath}${upload}1`, '-v'], httpCode => { assert.strictEqual(httpCode, '200 OK'); done(); }); }); - it('should put second object in existing bucket with prefix ' + - 'and delimiter', done => { - provideRawOutput( - [`--put=${upload}`, '--', `${prefixedPath}${upload}2`, '-v'], - httpCode => { - assert.strictEqual(httpCode, '200 OK'); - done(); - }); + it('should put second object in existing bucket with prefix ' + 'and delimiter', done => { + provideRawOutput([`--put=${upload}`, '--', `${prefixedPath}${upload}2`, '-v'], httpCode => { + assert.strictEqual(httpCode, '200 OK'); + done(); + }); }); - it('should put third object in existing bucket with prefix ' + - 'and delimiter', done => { - provideRawOutput([ - `--put=${upload}`, - '--', - `${prefixedPath}${upload}3`, - '-v', - ], httpCode => { + it('should put third object in existing bucket with prefix ' + 'and delimiter', done => { + provideRawOutput([`--put=${upload}`, '--', `${prefixedPath}${upload}3`, '-v'], httpCode => { assert.strictEqual(httpCode, '200 OK'); done(); }); @@ -581,21 +496,15 @@ describe('s3curl putObject', () => { }); describe('s3curl getBucket', () => { - const objects = [ - `${prefixedPath}${upload}1`, - `${prefixedPath}${upload}2`, - `${prefixedPath}${upload}3`, - ]; + const objects = [`${prefixedPath}${upload}1`, `${prefixedPath}${upload}2`, `${prefixedPath}${upload}3`]; before(done => { - provideRawOutput( - ['--createBucket', '--', bucketPath, '-v'], - httpCode => { - assert.strictEqual(httpCode, '200 OK'); - createFile(upload, 1048576, () => { - putObjects(upload, objects, done); - }); + provideRawOutput(['--createBucket', '--', bucketPath, '-v'], httpCode => { + assert.strictEqual(httpCode, '200 OK'); + createFile(upload, 1048576, () => { + putObjects(upload, objects, done); }); + }); }); after(done => { @@ -604,143 +513,114 @@ describe('s3curl getBucket', () => { }); it('should list all objects if no prefix or delimiter specified', done => { - provideRawOutput( - ['--', bucketPath, '-v'], - (httpCode, rawOutput) => { - assert.strictEqual(httpCode, '200 OK'); - parseString(rawOutput.stdout, (err, result) => { - if (err) { - assert.ifError(err); - } - assert.strictEqual(result.ListBucketResult - .Contents[0].Key[0], `${basePath}${upload}1`); - assert.strictEqual(result.ListBucketResult - .Contents[1].Key[0], `${basePath}${upload}2`); - assert.strictEqual(result.ListBucketResult - .Contents[2].Key[0], `${basePath}${upload}3`); - done(); - }); - }); - }); - - it('should list a common prefix if a common prefix and delimiter are ' + - 'specified', done => { - provideRawOutput([ - '--', - `${bucketPath}?delimiter=${delimiter}&prefix=${prefix}`, - '-v', - ], (httpCode, rawOutput) => { + provideRawOutput(['--', bucketPath, '-v'], (httpCode, rawOutput) => { assert.strictEqual(httpCode, '200 OK'); parseString(rawOutput.stdout, (err, result) => { if (err) { assert.ifError(err); } - assert.strictEqual(result.ListBucketResult - .CommonPrefixes[0].Prefix[0], basePath); + assert.strictEqual(result.ListBucketResult.Contents[0].Key[0], `${basePath}${upload}1`); + assert.strictEqual(result.ListBucketResult.Contents[1].Key[0], `${basePath}${upload}2`); + assert.strictEqual(result.ListBucketResult.Contents[2].Key[0], `${basePath}${upload}3`); done(); }); }); }); - it('should not list a common prefix if no delimiter is specified', done => { + it('should list a common prefix if a common prefix and delimiter are ' + 'specified', done => { provideRawOutput( - ['--', `${bucketPath}?&prefix=${prefix}`, '-v'], + ['--', `${bucketPath}?delimiter=${delimiter}&prefix=${prefix}`, '-v'], (httpCode, rawOutput) => { assert.strictEqual(httpCode, '200 OK'); parseString(rawOutput.stdout, (err, result) => { if (err) { assert.ifError(err); } - const keys = Object.keys(result.ListBucketResult); - const location = keys.indexOf('CommonPrefixes'); - assert.strictEqual(location, -1); - assert.strictEqual(result.ListBucketResult - .Contents[0].Key[0], `${basePath}${upload}1`); + assert.strictEqual(result.ListBucketResult.CommonPrefixes[0].Prefix[0], basePath); done(); }); + }, + ); + }); + + it('should not list a common prefix if no delimiter is specified', done => { + provideRawOutput(['--', `${bucketPath}?&prefix=${prefix}`, '-v'], (httpCode, rawOutput) => { + assert.strictEqual(httpCode, '200 OK'); + parseString(rawOutput.stdout, (err, result) => { + if (err) { + assert.ifError(err); + } + const keys = Object.keys(result.ListBucketResult); + const location = keys.indexOf('CommonPrefixes'); + assert.strictEqual(location, -1); + assert.strictEqual(result.ListBucketResult.Contents[0].Key[0], `${basePath}${upload}1`); + done(); }); + }); }); - it('should provide a next marker if maxs keys exceeded ' + - 'and delimiter specified', done => { - provideRawOutput( - ['--', `${bucketPath}?delimiter=x&max-keys=2`, '-v'], - (httpCode, rawOutput) => { - assert.strictEqual(httpCode, '200 OK'); - parseString(rawOutput.stdout, (err, result) => { - if (err) { - assert.ifError(err); - } - assert.strictEqual(result.ListBucketResult - .NextMarker[0], `${basePath}${upload}2`); - assert.strictEqual(result.ListBucketResult - .IsTruncated[0], 'true'); - done(); - }); + it('should provide a next marker if maxs keys exceeded ' + 'and delimiter specified', done => { + provideRawOutput(['--', `${bucketPath}?delimiter=x&max-keys=2`, '-v'], (httpCode, rawOutput) => { + assert.strictEqual(httpCode, '200 OK'); + parseString(rawOutput.stdout, (err, result) => { + if (err) { + assert.ifError(err); + } + assert.strictEqual(result.ListBucketResult.NextMarker[0], `${basePath}${upload}2`); + assert.strictEqual(result.ListBucketResult.IsTruncated[0], 'true'); + done(); }); + }); }); it('should return InvalidArgument error with negative max-keys', done => { - provideRawOutput( - ['--', `${bucketPath}?&max-keys=-2`, '-v'], - (httpCode, rawOutput) => { - assert.strictEqual(httpCode, '400 BAD REQUEST'); - assertError(rawOutput.stdout, 'InvalidArgument', done); - }); + provideRawOutput(['--', `${bucketPath}?&max-keys=-2`, '-v'], (httpCode, rawOutput) => { + assert.strictEqual(httpCode, '400 BAD REQUEST'); + assertError(rawOutput.stdout, 'InvalidArgument', done); + }); }); it('should return InvalidArgument error with invalid max-keys', done => { - provideRawOutput( - ['--', `${bucketPath}?max-keys='slash'`, '-v'], - (httpCode, rawOutput) => { - assert.strictEqual(httpCode, '400 BAD REQUEST'); - assertError(rawOutput.stdout, 'InvalidArgument', done); - }); + provideRawOutput(['--', `${bucketPath}?max-keys='slash'`, '-v'], (httpCode, rawOutput) => { + assert.strictEqual(httpCode, '400 BAD REQUEST'); + assertError(rawOutput.stdout, 'InvalidArgument', done); + }); }); it('should return an EncodingType XML tag with the value "url"', done => { - provideRawOutput( - ['--', bucketPath, '-G', '-d', 'encoding-type=url', '-v'], - (httpCode, rawOutput) => { - assert.strictEqual(httpCode, '200 OK'); - parseString(rawOutput.stdout, (err, result) => { - if (err) { - assert.ifError(err); - } - assert.strictEqual(result.ListBucketResult - .EncodingType[0], 'url'); - done(); - }); + provideRawOutput(['--', bucketPath, '-G', '-d', 'encoding-type=url', '-v'], (httpCode, rawOutput) => { + assert.strictEqual(httpCode, '200 OK'); + parseString(rawOutput.stdout, (err, result) => { + if (err) { + assert.ifError(err); + } + assert.strictEqual(result.ListBucketResult.EncodingType[0], 'url'); + done(); }); + }); }); - it('should return an InvalidArgument error when given an invalid ' + - 'encoding type', done => { - provideRawOutput( - ['--', bucketPath, '-G', '-d', 'encoding-type=invalidURI', '-v'], - (httpCode, rawOutput) => { - assert.strictEqual(httpCode, '400 BAD REQUEST'); - parseString(rawOutput.stdout, (err, result) => { - if (err) { - assert.ifError(err); - } - assert.strictEqual(result.Error.Code[0], 'InvalidArgument'); - assert.strictEqual(result.Error.Message[0], - 'Invalid Encoding Method specified in Request'); - done(); - }); + it('should return an InvalidArgument error when given an invalid ' + 'encoding type', done => { + provideRawOutput(['--', bucketPath, '-G', '-d', 'encoding-type=invalidURI', '-v'], (httpCode, rawOutput) => { + assert.strictEqual(httpCode, '400 BAD REQUEST'); + parseString(rawOutput.stdout, (err, result) => { + if (err) { + assert.ifError(err); + } + assert.strictEqual(result.Error.Code[0], 'InvalidArgument'); + assert.strictEqual(result.Error.Message[0], 'Invalid Encoding Method specified in Request'); + done(); }); + }); }); }); describe('s3curl head bucket', () => { before(done => { - provideRawOutput( - ['--createBucket', '--', bucketPath, '-v'], - httpCode => { - assert.strictEqual(httpCode, '200 OK'); - done(); - }); + provideRawOutput(['--createBucket', '--', bucketPath, '-v'], httpCode => { + assert.strictEqual(httpCode, '200 OK'); + done(); + }); }); after(done => { @@ -748,78 +628,59 @@ describe('s3curl head bucket', () => { }); it('should return a 404 response if bucket does not exist', done => { - provideRawOutput( - ['--head', '--', `${endpoint}/${nonexist}`, '-v'], - httpCode => { - assert.strictEqual(httpCode, '404 NOT FOUND'); - done(); - }); + provideRawOutput(['--head', '--', `${endpoint}/${nonexist}`, '-v'], httpCode => { + assert.strictEqual(httpCode, '404 NOT FOUND'); + done(); + }); }); - it('should return a 200 response if bucket exists' + - ' and user is authorized', done => { - provideRawOutput( - ['--head', '--', bucketPath, '-v'], - httpCode => { - assert.strictEqual(httpCode, '200 OK'); - done(); - }); + it('should return a 200 response if bucket exists' + ' and user is authorized', done => { + provideRawOutput(['--head', '--', bucketPath, '-v'], httpCode => { + assert.strictEqual(httpCode, '200 OK'); + done(); + }); }); }); describe('s3curl getObject', () => { before(done => { createFile(upload, 1048576, () => { - provideRawOutput( - ['--createBucket', '--', bucketPath, '-v'], - httpCode => { - assert.strictEqual(httpCode, '200 OK'); - done(); - }); + provideRawOutput(['--createBucket', '--', bucketPath, '-v'], httpCode => { + assert.strictEqual(httpCode, '200 OK'); + done(); + }); }); }); after('delete created file and downloaded file', done => { - const objects = [ - `${bucketPath}/getter`, - bucketPath, - ]; + const objects = [`${bucketPath}/getter`, bucketPath]; deleteRemoteItems(objects, () => { deleteFile(upload, () => deleteFile(download, done)); }); }); it('should put object with metadata', done => { - provideRawOutput([ - `--put=${upload}`, - '--', - '-H', - 'x-amz-meta-mine:BestestObjectEver', - `${bucketPath}/getter`, - '-v', - ], httpCode => { - assert.strictEqual(httpCode, '200 OK'); - done(); - }); - }); - - it('should get an existing file in an existing bucket', done => { provideRawOutput( - ['--', '-o', download, `${bucketPath}/getter`, '-v'], + [`--put=${upload}`, '--', '-H', 'x-amz-meta-mine:BestestObjectEver', `${bucketPath}/getter`, '-v'], httpCode => { assert.strictEqual(httpCode, '200 OK'); done(); - }); + }, + ); }); - it('should return an error if getting object with empty bucket name', - done => { - provideRawOutput( - ['--', '-o', download, `${endpoint}//getter`, '-v'], - httpCode => { - assert.strictEqual(httpCode, '405 METHOD NOT ALLOWED'); - done(); - }); + it('should get an existing file in an existing bucket', done => { + provideRawOutput(['--', '-o', download, `${bucketPath}/getter`, '-v'], httpCode => { + assert.strictEqual(httpCode, '200 OK'); + done(); + }); + }); + + it('should return an error if getting object with empty bucket name', done => { + provideRawOutput(['--', '-o', download, `${endpoint}//getter`, '-v'], httpCode => { + assert.strictEqual(httpCode, '405 METHOD NOT ALLOWED'); + done(); + }); }); it.skip('downloaded file should equal uploaded file', done => { @@ -830,160 +691,135 @@ describe('s3curl getObject', () => { describe('s3curl head object', () => { before(done => { createFile(upload, 1048576, () => { - provideRawOutput( - ['--createBucket', '--', bucketPath, '-v'], - httpCode => { - assert.strictEqual(httpCode, '200 OK'); - provideRawOutput([ - `--put=${upload}`, - '--', - '-H', - 'x-amz-meta-mine:BestestObjectEver', - `${bucketPath}/getter`, - '-v', - ], httpCode => { + provideRawOutput(['--createBucket', '--', bucketPath, '-v'], httpCode => { + assert.strictEqual(httpCode, '200 OK'); + provideRawOutput( + [`--put=${upload}`, '--', '-H', 'x-amz-meta-mine:BestestObjectEver', `${bucketPath}/getter`, '-v'], + httpCode => { assert.strictEqual(httpCode, '200 OK'); done(); - }); - }); + }, + ); + }); }); }); after(done => { - deleteRemoteItems([ - `${bucketPath}/getter`, - bucketPath, - ], done); + deleteRemoteItems([`${bucketPath}/getter`, bucketPath], done); }); it("should get object's metadata", done => { - provideRawOutput( - ['--head', '--', `${bucketPath}/getter`, '-v'], - (httpCode, rawOutput) => { - assert.strictEqual(httpCode, '200 OK'); - const lines = rawOutput.stdout.split('\n'); - const userMetadata = 'x-amz-meta-mine: BestestObjectEver\r'; - assert(lines.indexOf(userMetadata) > -1); - assert(rawOutput.stdout.indexOf('ETag') > -1); - done(); - }); + provideRawOutput(['--head', '--', `${bucketPath}/getter`, '-v'], (httpCode, rawOutput) => { + assert.strictEqual(httpCode, '200 OK'); + const lines = rawOutput.stdout.split('\n'); + const userMetadata = 'x-amz-meta-mine: BestestObjectEver\r'; + assert(lines.indexOf(userMetadata) > -1); + assert(rawOutput.stdout.indexOf('ETag') > -1); + done(); + }); }); }); describe('s3curl object ACLs', () => { before(done => { createFile(aclUpload, 512000, () => { - provideRawOutput( - ['--createBucket', '--', bucketPath, '-v'], - httpCode => { - assert.strictEqual(httpCode, '200 OK'); - done(); - }); + provideRawOutput(['--createBucket', '--', bucketPath, '-v'], httpCode => { + assert.strictEqual(httpCode, '200 OK'); + done(); + }); }); }); after(done => { - deleteRemoteItems([ - `${bucketPath}/${aclUpload}withcannedacl`, - `${bucketPath}/${aclUpload}withspecificacl`, - bucketPath, - ], () => deleteFile(aclUpload, done)); + deleteRemoteItems( + [`${bucketPath}/${aclUpload}withcannedacl`, `${bucketPath}/${aclUpload}withspecificacl`, bucketPath], + () => deleteFile(aclUpload, done), + ); }); it('should put an object with a canned ACL', done => { - provideRawOutput([ - `--put=${aclUpload}`, - '--', - '-H', - 'x-amz-acl:public-read', - `${bucketPath}/${aclUpload}withcannedacl`, - '-v', - ], httpCode => { - assert.strictEqual(httpCode, '200 OK'); - done(); - }); + provideRawOutput( + [ + `--put=${aclUpload}`, + '--', + '-H', + 'x-amz-acl:public-read', + `${bucketPath}/${aclUpload}withcannedacl`, + '-v', + ], + httpCode => { + assert.strictEqual(httpCode, '200 OK'); + done(); + }, + ); }); it("should get an object's canned ACL", done => { - provideRawOutput([ - '--', - `${bucketPath}/${aclUpload}withcannedacl?acl`, - '-v', - ], (httpCode, rawOutput) => { + provideRawOutput(['--', `${bucketPath}/${aclUpload}withcannedacl?acl`, '-v'], (httpCode, rawOutput) => { assert.strictEqual(httpCode, '200 OK'); parseString(rawOutput.stdout, (err, result) => { if (err) { assert.ifError(err); } - assert.strictEqual(result.AccessControlPolicy - .Owner[0].ID[0], ownerCanonicalId); - assert.strictEqual(result.AccessControlPolicy - .AccessControlList[0].Grant[0] - .Grantee[0].ID[0], ownerCanonicalId); - assert.strictEqual(result.AccessControlPolicy - .AccessControlList[0].Grant[0] - .Permission[0], 'FULL_CONTROL'); - assert.strictEqual(result.AccessControlPolicy - .AccessControlList[0].Grant[1] - .Grantee[0].URI[0], - 'http://acs.amazonaws.com/groups/global/AllUsers'); - assert.strictEqual(result.AccessControlPolicy - .AccessControlList[0].Grant[1] - .Permission[0], 'READ'); + assert.strictEqual(result.AccessControlPolicy.Owner[0].ID[0], ownerCanonicalId); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Grantee[0].ID[0], + ownerCanonicalId, + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Permission[0], + 'FULL_CONTROL', + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[1].Grantee[0].URI[0], + 'http://acs.amazonaws.com/groups/global/AllUsers', + ); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[1].Permission[0], 'READ'); done(); }); }); }); it('should put an object with a specific ACL', done => { - provideRawOutput([ - `--put=${aclUpload}`, - '--', - '-H', - 'x-amz-grant-read:uri=' + - 'http://acs.amazonaws.com/groups/global/AuthenticatedUsers', - `${bucketPath}/${aclUpload}withspecificacl`, - '-v', - ], httpCode => { - assert.strictEqual(httpCode, '200 OK'); - done(); - }); + provideRawOutput( + [ + `--put=${aclUpload}`, + '--', + '-H', + 'x-amz-grant-read:uri=' + 'http://acs.amazonaws.com/groups/global/AuthenticatedUsers', + `${bucketPath}/${aclUpload}withspecificacl`, + '-v', + ], + httpCode => { + assert.strictEqual(httpCode, '200 OK'); + done(); + }, + ); }); it("should get an object's specific ACL", done => { - provideRawOutput([ - '--', - `${bucketPath}/${aclUpload}withspecificacl?acl`, - '-v', - ], (httpCode, rawOutput) => { + provideRawOutput(['--', `${bucketPath}/${aclUpload}withspecificacl?acl`, '-v'], (httpCode, rawOutput) => { assert.strictEqual(httpCode, '200 OK'); parseString(rawOutput.stdout, (err, result) => { if (err) { assert.ifError(err); } - assert.strictEqual(result.AccessControlPolicy - .Owner[0].ID[0], ownerCanonicalId); - assert.strictEqual(result.AccessControlPolicy - .AccessControlList[0].Grant[0] - .Grantee[0].URI[0], - 'http://acs.amazonaws.com/groups/global/' + - 'AuthenticatedUsers'); - assert.strictEqual(result.AccessControlPolicy - .AccessControlList[0].Grant[0] - .Permission[0], 'READ'); + assert.strictEqual(result.AccessControlPolicy.Owner[0].ID[0], ownerCanonicalId); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Grantee[0].URI[0], + 'http://acs.amazonaws.com/groups/global/' + 'AuthenticatedUsers', + ); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[0].Permission[0], 'READ'); done(); }); }); }); - it('should return a NoSuchKey error if try to get an object' + - 'ACL for an object that does not exist', done => { - provideRawOutput( - ['--', `${bucketPath}/keydoesnotexist?acl`, '-v'], - (httpCode, rawOutput) => { - assert.strictEqual(httpCode, '404 NOT FOUND'); - assertError(rawOutput.stdout, 'NoSuchKey', done); - }); + it('should return a NoSuchKey error if try to get an object' + 'ACL for an object that does not exist', done => { + provideRawOutput(['--', `${bucketPath}/keydoesnotexist?acl`, '-v'], (httpCode, rawOutput) => { + assert.strictEqual(httpCode, '404 NOT FOUND'); + assertError(rawOutput.stdout, 'NoSuchKey', done); + }); }); }); @@ -993,104 +829,70 @@ describe('s3curl multipart upload', () => { let uploadId = null; before(done => { - provideRawOutput( - ['--createBucket', '--', bucketPath, '-v'], - httpCode => { - assert.strictEqual(httpCode, '200 OK'); - // initiate mpu - provideRawOutput([ - '--', - '-X', - 'POST', - `${bucketPath}/${key}?uploads`, - '-v', - ], (httpCode, rawOutput) => { - parseString(rawOutput.stdout, (err, result) => { - if (err) { - assert.ifError(err); - } - uploadId = - result.InitiateMultipartUploadResult.UploadId[0]; - // create file to copy - createFile(upload, 100, () => { - // put file to copy - putObjects(upload, [`${bucketPath}/copyme`], done); - }); + provideRawOutput(['--createBucket', '--', bucketPath, '-v'], httpCode => { + assert.strictEqual(httpCode, '200 OK'); + // initiate mpu + provideRawOutput(['--', '-X', 'POST', `${bucketPath}/${key}?uploads`, '-v'], (httpCode, rawOutput) => { + parseString(rawOutput.stdout, (err, result) => { + if (err) { + assert.ifError(err); + } + uploadId = result.InitiateMultipartUploadResult.UploadId[0]; + // create file to copy + createFile(upload, 100, () => { + // put file to copy + putObjects(upload, [`${bucketPath}/copyme`], done); }); }); }); + }); }); after(done => { - deleteRemoteItems([ - `${bucketPath}/copyme`, - `${bucketPath}/${key}?uploadId=${uploadId}`, - bucketPath, - ], () => deleteFile(upload, done)); + deleteRemoteItems([`${bucketPath}/copyme`, `${bucketPath}/${key}?uploadId=${uploadId}`, bucketPath], () => + deleteFile(upload, done), + ); }); it('should return error for list parts call if no key sent', done => { - provideRawOutput([ - '--', - `${bucketPath}?uploadId=${uploadId}`, - '-v', - ], (httpCode, rawOutput) => { + provideRawOutput(['--', `${bucketPath}?uploadId=${uploadId}`, '-v'], (httpCode, rawOutput) => { assert.strictEqual(httpCode, '400 BAD REQUEST'); assertError(rawOutput.stdout, 'InvalidRequest', done); }); }); it('should return error for put part call if no key sent', done => { - provideRawOutput([ - '--', - '-X', 'PUT', - `${bucketPath}?partNumber=1&uploadId=${uploadId}`, - '-v', - ], (httpCode, rawOutput) => { - assert.strictEqual(httpCode, '400 BAD REQUEST'); - assertError(rawOutput.stdout, 'InvalidRequest', done); - }); + provideRawOutput( + ['--', '-X', 'PUT', `${bucketPath}?partNumber=1&uploadId=${uploadId}`, '-v'], + (httpCode, rawOutput) => { + assert.strictEqual(httpCode, '400 BAD REQUEST'); + assertError(rawOutput.stdout, 'InvalidRequest', done); + }, + ); }); it('should return error for complete mpu call if no key sent', done => { - provideRawOutput([ - '--', - '-X', 'POST', - `${bucketPath}?uploadId=${uploadId}`, - '-v', - ], (httpCode, rawOutput) => { + provideRawOutput(['--', '-X', 'POST', `${bucketPath}?uploadId=${uploadId}`, '-v'], (httpCode, rawOutput) => { assert.strictEqual(httpCode, '400 BAD REQUEST'); assertError(rawOutput.stdout, 'InvalidRequest', done); }); }); it('should return error for abort mpu call if no key sent', done => { - provideRawOutput([ - '--', - '-X', 'DELETE', - `${bucketPath}?uploadId=${uploadId}`, - '-v', - ], (httpCode, rawOutput) => { + provideRawOutput(['--', '-X', 'DELETE', `${bucketPath}?uploadId=${uploadId}`, '-v'], (httpCode, rawOutput) => { assert.strictEqual(httpCode, '400 BAD REQUEST'); assertError(rawOutput.stdout, 'InvalidRequest', done); }); }); it('should list parts of multipart upload with no parts', done => { - provideRawOutput([ - '--', - `${bucketPath}/${key}?uploadId=${uploadId}`, - '-v', - ], (httpCode, rawOutput) => { + provideRawOutput(['--', `${bucketPath}/${key}?uploadId=${uploadId}`, '-v'], (httpCode, rawOutput) => { assert.strictEqual(httpCode, '200 OK'); parseString(rawOutput.stdout, (err, result) => { - assert.strictEqual(result.ListPartsResult.UploadId[0], - uploadId); - assert.strictEqual(result.ListPartsResult.Bucket[0], - bucket); + assert.strictEqual(result.ListPartsResult.UploadId[0], uploadId); + assert.strictEqual(result.ListPartsResult.Bucket[0], bucket); assert.strictEqual(result.ListPartsResult.Key[0], key); - assert.strictEqual(result.ListPartsResult.Part, - undefined); + assert.strictEqual(result.ListPartsResult.Part, undefined); done(); }); }); @@ -1098,68 +900,76 @@ describe('s3curl multipart upload', () => { it('should copy a part and return lastModified as ISO', done => { provideRawOutput( - ['--', `${bucketPath}/${key}?uploadId=${uploadId}&partNumber=1`, - '-X', 'PUT', '-H', - `x-amz-copy-source:${bucket}/copyme`, '-v'], + [ + '--', + `${bucketPath}/${key}?uploadId=${uploadId}&partNumber=1`, + '-X', + 'PUT', + '-H', + `x-amz-copy-source:${bucket}/copyme`, + '-v', + ], (httpCode, rawOutput) => { assert.strictEqual(httpCode, '200 OK'); parseString(rawOutput.stdout, (err, result) => { - const lastModified = result.CopyPartResult - .LastModified[0]; + const lastModified = result.CopyPartResult.LastModified[0]; const isoDateString = new Date(lastModified).toISOString(); assert.strictEqual(lastModified, isoDateString); done(); }); - }); + }, + ); }); }); describe('s3curl copy object', () => { before(done => { createFile(upload, 1048576, () => { - provideRawOutput( - ['--createBucket', '--', bucketPath, '-v'], - httpCode => { - assert.strictEqual(httpCode, '200 OK'); - putObjects(upload, [`${bucketPath}/copyme`], done); - }); + provideRawOutput(['--createBucket', '--', bucketPath, '-v'], httpCode => { + assert.strictEqual(httpCode, '200 OK'); + putObjects(upload, [`${bucketPath}/copyme`], done); + }); }); }); after(done => { - deleteRemoteItems([ - `${bucketPath}/copyme`, - `${bucketPath}/iamacopy`, - bucketPath, - ], () => deleteFile(upload, done)); + deleteRemoteItems([`${bucketPath}/copyme`, `${bucketPath}/iamacopy`, bucketPath], () => + deleteFile(upload, done), + ); }); it('should copy an object and return lastModified as ISO', done => { provideRawOutput( - ['--', `${bucketPath}/iamacopy`, '-X', 'PUT', '-H', - `x-amz-copy-source:${bucket}/copyme`, '-v'], + ['--', `${bucketPath}/iamacopy`, '-X', 'PUT', '-H', `x-amz-copy-source:${bucket}/copyme`, '-v'], (httpCode, rawOutput) => { assert.strictEqual(httpCode, '200 OK'); parseString(rawOutput.stdout, (err, result) => { - const lastModified = result.CopyObjectResult - .LastModified[0]; + const lastModified = result.CopyObjectResult.LastModified[0]; const isoDateString = new Date(lastModified).toISOString(); assert.strictEqual(lastModified, isoDateString); done(); }); - }); + }, + ); }); }); describe('s3curl multi-object delete', () => { it('should return an error if md5 is wrong', done => { - provideRawOutput(['--post', 'multiDelete.xml', '--contentMd5', - 'p5/WA/oEr30qrEEl21PAqw==', '--', - `${endpoint}/${bucket}/?delete`, '-v'], + provideRawOutput( + [ + '--post', + 'multiDelete.xml', + '--contentMd5', + 'p5/WA/oEr30qrEEl21PAqw==', + '--', + `${endpoint}/${bucket}/?delete`, + '-v', + ], (httpCode, rawOutput) => { assert.strictEqual(httpCode, '400 BAD REQUEST'); - assertError(rawOutput.stdout, 'BadDigest', - done); - }); + assertError(rawOutput.stdout, 'BadDigest', done); + }, + ); }); }); diff --git a/tests/functional/sse-kms-migration/arnPrefix.js b/tests/functional/sse-kms-migration/arnPrefix.js index 5a70ed5e97..60231be12a 100644 --- a/tests/functional/sse-kms-migration/arnPrefix.js +++ b/tests/functional/sse-kms-migration/arnPrefix.js @@ -30,12 +30,10 @@ describe('SSE KMS arnPrefix', () => { bkts[bktConf.name] = bkt; if (bktConf.algo && bktConf.masterKeyId) { bkt.kmsKeyInfo = await helpers.createKmsKey(log); - bkt.kmsKey = bktConf.arnPrefix - ? bkt.kmsKeyInfo.masterKeyArn - : bkt.kmsKeyInfo.masterKeyId; + bkt.kmsKey = bktConf.arnPrefix ? bkt.kmsKeyInfo.masterKeyArn : bkt.kmsKeyInfo.masterKeyId; } - await helpers.s3.createBucket(({ Bucket: bkt.name })); - await helpers.s3.createBucket(({ Bucket: bkt.vname })); + await helpers.s3.createBucket({ Bucket: bkt.name }); + await helpers.s3.createBucket({ Bucket: bkt.vname }); if (bktConf.deleteSSE) { await scenarios.deleteBucketSSEBeforeEach(bkt.name, log); await scenarios.deleteBucketSSEBeforeEach(bkt.vname, log); @@ -45,38 +43,44 @@ describe('SSE KMS arnPrefix', () => { await helpers.s3.putBucketEncryption({ Bucket: bkt.name, ServerSideEncryptionConfiguration: helpers.hydrateSSEConfig({ - algo: bktConf.algo, masterKeyId: bkt.kmsKey }), + algo: bktConf.algo, + masterKeyId: bkt.kmsKey, + }), }); await helpers.s3.putBucketEncryption({ Bucket: bkt.vname, ServerSideEncryptionConfiguration: helpers.hydrateSSEConfig({ - algo: bktConf.algo, masterKeyId: bkt.kmsKey }), + algo: bktConf.algo, + masterKeyId: bkt.kmsKey, + }), }); } // Put an object for each SSE conf in each bucket - await Promise.all(scenarios.testCases.map(async objConf => { - const obj = { - name: `for-copy-enc-obj-${objConf.name}`, - kmsKeyInfo: null, - kmsKey: null, - body: `BODY(for-copy-enc-obj-${objConf.name})`, - }; - bkt.objs[objConf.name] = obj; - if (objConf.algo && objConf.masterKeyId) { - obj.kmsKeyInfo = await helpers.createKmsKey(log); - obj.kmsKey = objConf.arnPrefix - ? obj.kmsKeyInfo.masterKeyArn - : obj.kmsKeyInfo.masterKeyId; - } + await Promise.all( + scenarios.testCases.map(async objConf => { + const obj = { + name: `for-copy-enc-obj-${objConf.name}`, + kmsKeyInfo: null, + kmsKey: null, + body: `BODY(for-copy-enc-obj-${objConf.name})`, + }; + bkt.objs[objConf.name] = obj; + if (objConf.algo && objConf.masterKeyId) { + obj.kmsKeyInfo = await helpers.createKmsKey(log); + obj.kmsKey = objConf.arnPrefix ? obj.kmsKeyInfo.masterKeyArn : obj.kmsKeyInfo.masterKeyId; + } - return await helpers.putEncryptedObject(bkt.name, obj.name, objConf, obj.kmsKey, obj.body); - })); + return await helpers.putEncryptedObject(bkt.name, obj.name, objConf, obj.kmsKey, obj.body); + }), + ); }; before('setup', async () => { - console.log('Run arnPrefix', - { profile: helpers.credsProfile, accessKeyId: helpers.s3.config.credentials.accessKeyId }); + console.log('Run arnPrefix', { + profile: helpers.credsProfile, + accessKeyId: helpers.s3.config.credentials.accessKeyId, + }); const allBuckets = (await helpers.s3.listBuckets()).Buckets.map(b => b.Name); console.log('List buckets:', allBuckets); await helpers.MD.setup(); @@ -85,15 +89,19 @@ describe('SSE KMS arnPrefix', () => { // pre cleanup await helpers.cleanup(copyBkt); await helpers.cleanup(mpuCopyBkt); - await Promise.all(Object.values(bkts).map(async bkt => { - await helpers.cleanup(bkt.name); - return await helpers.cleanup(bkt.vname); - })); - } catch (e) { void e; } + await Promise.all( + Object.values(bkts).map(async bkt => { + await helpers.cleanup(bkt.name); + return await helpers.cleanup(bkt.vname); + }), + ); + } catch (e) { + void e; + } // init copy bucket - await helpers.s3.createBucket(({ Bucket: copyBkt })); - await helpers.s3.createBucket(({ Bucket: mpuCopyBkt })); + await helpers.s3.createBucket({ Bucket: copyBkt }); + await helpers.s3.createBucket({ Bucket: mpuCopyBkt }); await helpers.s3.putBucketEncryption({ Bucket: copyBkt, ServerSideEncryptionConfiguration: helpers.hydrateSSEConfig({ algo: 'aws:kms', masterKeyId: copyKmsKey }), @@ -108,259 +116,353 @@ describe('SSE KMS arnPrefix', () => { await helpers.cleanup(copyBkt); await helpers.cleanup(mpuCopyBkt); // Clean every bucket - await Promise.all(Object.values(bkts).map(async bkt => { - await helpers.cleanup(bkt.name); - return await helpers.cleanup(bkt.vname); - })); + await Promise.all( + Object.values(bkts).map(async bkt => { + await helpers.cleanup(bkt.name); + return await helpers.cleanup(bkt.vname); + }), + ); }); - scenarios.testCases.forEach(bktConf => describe(`bucket enc-bkt-${bktConf.name}`, () => { - let bkt = bkts[bktConf.name]; - - before(() => { - bkt = bkts[bktConf.name]; - }); + scenarios.testCases.forEach(bktConf => + describe(`bucket enc-bkt-${bktConf.name}`, () => { + let bkt = bkts[bktConf.name]; - if (bktConf.deleteSSE) { - beforeEach(async () => { - await scenarios.deleteBucketSSEBeforeEach(bkt.name, log); - await scenarios.deleteBucketSSEBeforeEach(bkt.vname, log); + before(() => { + bkt = bkts[bktConf.name]; }); - } - if (!bktConf.algo) { - if (!bktConf.deleteSSE && helpers.config.globalEncryptionEnabled) { - it('GetBucketEncryption should return AES256 because of globalEncryptionEnabled', - async () => await scenarios.tests.getBucketSSE(bkt.name, log, 'AES256', null, 'after')); - } else { - it('GetBucketEncryption should return ServerSideEncryptionConfigurationNotFoundError', - async () => await scenarios.tests.getBucketSSEError(bkt.name)); - if (!bktConf.deleteSSE) { - it('should have non mandatory SSE in bucket MD as test init put an object with AES256', - async () => await scenarios.tests.getBucketNonMandatorySSE(bkt.name, log, 'after')); - } + if (bktConf.deleteSSE) { + beforeEach(async () => { + await scenarios.deleteBucketSSEBeforeEach(bkt.name, log); + await scenarios.deleteBucketSSEBeforeEach(bkt.vname, log); + }); } - } else { - it('GetBucketEncryption should return SSE with arnPrefix to key', - async () => await scenarios.tests.getBucketSSE(bkt.name, log, bktConf.algo, - bktConf.masterKeyId ? bkt.kmsKeyInfo.masterKeyArn : null, 'after')); - } - scenarios.testCasesObj.forEach(objConf => it(`should assert uploaded objects with SSE ${objConf.name}`, - async () => scenarios.tests.getPreUploadedObject(bkt.name, - { objConf, obj: bkt.objs[objConf.name] }, { bktConf, bkt }, 'after'))); - - scenarios.testCasesObj.forEach(objConf => describe(`object enc-obj-${objConf.name}`, () => { - const obj = { - name: `enc-obj-${objConf.name}`, - kmsKeyInfo: null, - kmsKey: null, - body: `BODY(enc-obj-${objConf.name})`, - }; - /** to be used as source of copy */ - let objForCopy; - - before(async () => { - if (objConf.algo && objConf.masterKeyId) { - obj.kmsKeyInfo = await helpers.createKmsKey(log); - obj.kmsKey = objConf.arnPrefix - ? obj.kmsKeyInfo.masterKeyArn - : obj.kmsKeyInfo.masterKeyId; + if (!bktConf.algo) { + if (!bktConf.deleteSSE && helpers.config.globalEncryptionEnabled) { + it('GetBucketEncryption should return AES256 because of globalEncryptionEnabled', async () => + await scenarios.tests.getBucketSSE(bkt.name, log, 'AES256', null, 'after')); + } else { + it('GetBucketEncryption should return ServerSideEncryptionConfigurationNotFoundError', async () => + await scenarios.tests.getBucketSSEError(bkt.name)); + if (!bktConf.deleteSSE) { + it('should have non mandatory SSE in bucket MD as test init put an object with AES256', async () => + await scenarios.tests.getBucketNonMandatorySSE(bkt.name, log, 'after')); + } } - objForCopy = bkt.objs[objConf.name]; - }); + } else { + it('GetBucketEncryption should return SSE with arnPrefix to key', async () => + await scenarios.tests.getBucketSSE( + bkt.name, + log, + bktConf.algo, + bktConf.masterKeyId ? bkt.kmsKeyInfo.masterKeyArn : null, + 'after', + )); + } - it(`should PutObject ${obj.name} overriding bucket SSE`, - async () => scenarios.tests.putObjectOverrideSSE({ objConf, obj }, { bktConf, bkt }, 'after')); - - // CopyObject scenarios - [ - { name: `${obj.name} into encrypted destination bucket`, forceBktSSE: true }, - { name: `${obj.name} into same bucket with object SSE config` }, - { name: `from encrypted source into ${obj.name} with object SSE config` }, - ].forEach(({ name, forceBktSSE }, index) => - it(`should CopyObject ${name}`, async () => - await scenarios.tests.copyObjectAndSSE( - { copyBkt, objForCopy, copyObj }, - { objConf, obj }, + scenarios.testCasesObj.forEach(objConf => + it(`should assert uploaded objects with SSE ${objConf.name}`, async () => + scenarios.tests.getPreUploadedObject( + bkt.name, + { objConf, obj: bkt.objs[objConf.name] }, { bktConf, bkt }, - { index, forceBktSSE }, 'after', - ))); - - // after SSE migration implementation all mpu with sse are fixed - it('should encrypt MPU and put 2 encrypted parts', async () => { - const mpuKey = `${obj.name}-mpu`; - const mpu = await helpers.s3.createMultipartUpload( - helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey)); - const partsBody = [`${obj.body}-MPU1`, `${obj.body}-MPU2`]; - const newParts = []; - for (const [index, body] of partsBody.entries()) { - const part = await scenarios.tests.mpuUploadPart({ - UploadId: mpu.UploadId, - Bucket: bkt.name, - Body: body, - Key: mpuKey, - PartNumber: index + 1, - }, mpu, objConf.algo || bktConf.algo, 'after'); - newParts.push(part); - } - await scenarios.tests.mpuComplete( - { UploadId: mpu.UploadId, Bucket: bkt.name, Key: mpuKey }, - { existingParts: [], newParts }, - mpu, objConf.algo || bktConf.algo, 'after'); - const assertion = { - Bucket: bkt.name, - Key: mpuKey, - Body: `${obj.body}-MPU1${obj.body}-MPU2`, - }; - await scenarios.assertObjectSSE(assertion, { objConf, obj }, { bktConf, bkt }, {}, 'after'); - }); - - it('should encrypt MPU and copy an encrypted parts from encrypted bucket', async () => { - const mpuKey = `${obj.name}-mpucopy`; - const mpu = await helpers.s3.createMultipartUpload( - helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey)); - const part1 = await scenarios.tests.mpuUploadPartCopy({ - UploadId: mpu.UploadId, - Bucket: bkt.name, - Key: mpuKey, - PartNumber: 1, - CopySource: `${copyBkt}/${copyObj}`, - }, mpu, objConf.algo || bktConf.algo, 'after'); - const part2 = await scenarios.tests.mpuUploadPart({ - UploadId: mpu.UploadId, - Bucket: bkt.name, - Body: `${obj.body}-MPU2`, - Key: mpuKey, - PartNumber: 2, - }, mpu, objConf.algo || bktConf.algo, 'after'); - - await scenarios.tests.mpuComplete( - { UploadId: mpu.UploadId, Bucket: bkt.name, Key: mpuKey }, - { existingParts: [], newParts: [part1, part2] }, - mpu, objConf.algo || bktConf.algo, 'after'); - const assertion = { - Bucket: bkt.name, - Key: mpuKey, - Body: `BODY(copy)${obj.body}-MPU2`, - }; - await scenarios.assertObjectSSE(assertion, { objConf, obj }, { bktConf, bkt }, {}, 'after'); - }); - - it('should encrypt MPU and copy an encrypted range parts from encrypted bucket', async () => { - const mpuKey = `${obj.name}-mpucopy`; - const mpu = await helpers.s3.createMultipartUpload( - helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey)); - // source body is "BODY(copy)" - // [copy, BODY] - const sourceRanges = ['bytes=5-8', 'bytes=0-3']; - const newParts = []; - for (const [index, range] of sourceRanges.entries()) { - const part = await scenarios.tests.mpuUploadPartCopy({ - UploadId: mpu.UploadId, - Bucket: bkt.name, - Key: mpuKey, - PartNumber: index + 1, - CopySource: `${copyBkt}/${copyObj}`, - CopySourceRange: range, - }, mpu, objConf.algo || bktConf.algo, 'after'); - newParts.push(part); - } - - await scenarios.tests.mpuComplete( - { UploadId: mpu.UploadId, Bucket: bkt.name, Key: mpuKey }, - { existingParts: [], newParts }, - mpu, objConf.algo || bktConf.algo, 'after'); - const assertion = { - Bucket: bkt.name, - Key: mpuKey, - Body: 'copyBODY', - }; - await scenarios.assertObjectSSE(assertion, { objConf, obj }, { bktConf, bkt }, {}, 'after'); - }); - - it(`should PutObject versioned with SSE ${obj.name}`, async () => { - // ensure versioned bucket is empty - await helpers.bucketUtil.empty(bkt.vname); - let { Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname }); - // regularly count versioned objects - assert.strictEqual(Versions?.length, 0); - - const bodyBase = `BODY(${obj.name})-base`; - await helpers.putEncryptedObject(bkt.vname, obj.name, objConf, obj.kmsKey, bodyBase); - const baseAssertion = { Bucket: bkt.vname, Key: obj.name }; - await scenarios.assertObjectSSE( - { ...baseAssertion, Body: bodyBase }, - { objConf, obj }, { bktConf, bkt }, {}, 'after'); - ({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname })); - assert.strictEqual(Versions.length, 1); - - await helpers.s3.putBucketVersioning({ Bucket: bkt.vname, - VersioningConfiguration: { Status: 'Enabled' }, - }); - - const bodyV1 = `BODY(${obj.name})-v1`; - const v1 = await helpers.putEncryptedObject(bkt.vname, obj.name, objConf, obj.kmsKey, bodyV1); - const bodyV2 = `BODY(${obj.name})-v2`; - const v2 = await helpers.putEncryptedObject(bkt.vname, obj.name, objConf, obj.kmsKey, bodyV2); - ({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname })); - assert.strictEqual(Versions.length, 3); - - const current = await helpers.s3.headObject({ Bucket: bkt.vname, Key: obj.name }); - assert.strictEqual(current.VersionId, v2.VersionId); // ensure versioning as expected - ({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname })); - assert.strictEqual(Versions.length, 3); - - await scenarios.assertObjectSSE( - { ...baseAssertion, Body: bodyV2 }, { objConf, obj }, { bktConf, bkt }, - {}, 'after'); // v2 - await scenarios.assertObjectSSE( - { ...baseAssertion, VersionId: 'null', Body: bodyBase }, { objConf, obj }, { bktConf, bkt }, - {}, 'after'); - await scenarios.assertObjectSSE( - { ...baseAssertion, VersionId: v1.VersionId, Body: bodyV1 }, { objConf, obj }, { bktConf, bkt }, - {}, 'after'); - await scenarios.assertObjectSSE( - { ...baseAssertion, VersionId: v2.VersionId, Body: bodyV2 }, { objConf, obj }, { bktConf, bkt }, - {}, 'after'); - ({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname })); - assert.strictEqual(Versions.length, 3); - - await helpers.s3.putBucketVersioning({ Bucket: bkt.vname, - VersioningConfiguration: { Status: 'Suspended' }, - }); + )), + ); - // should be fine after version suspension - await scenarios.assertObjectSSE( - { ...baseAssertion, Body: bodyV2 }, { objConf, obj }, { bktConf, bkt }, - {}, 'after'); // v2 - await scenarios.assertObjectSSE( - { ...baseAssertion, VersionId: 'null', Body: bodyBase }, { objConf, obj }, { bktConf, bkt }, - {}, 'after'); - await scenarios.assertObjectSSE( - { ...baseAssertion, VersionId: v1.VersionId, Body: bodyV1 }, { objConf, obj }, { bktConf, bkt }, - {}, 'after'); - await scenarios.assertObjectSSE( - { ...baseAssertion, VersionId: v2.VersionId, Body: bodyV2 }, { objConf, obj }, { bktConf, bkt }, - {}, 'after'); - ({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname })); - assert.strictEqual(Versions.length, 3); - - // put a new null version - const bodyFinal = `BODY(${obj.name})-final`; - await helpers.putEncryptedObject(bkt.vname, obj.name, objConf, obj.kmsKey, bodyFinal); - await scenarios.assertObjectSSE( - { ...baseAssertion, Body: bodyFinal }, { objConf, obj }, { bktConf, bkt }, - {}, 'after'); // null - await scenarios.assertObjectSSE( - { ...baseAssertion, Body: bodyFinal }, { objConf, obj }, { bktConf, bkt }, - 'null', 'after'); - ({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname })); - assert.strictEqual(Versions.length, 3); - }); - })); - })); + scenarios.testCasesObj.forEach(objConf => + describe(`object enc-obj-${objConf.name}`, () => { + const obj = { + name: `enc-obj-${objConf.name}`, + kmsKeyInfo: null, + kmsKey: null, + body: `BODY(enc-obj-${objConf.name})`, + }; + /** to be used as source of copy */ + let objForCopy; + + before(async () => { + if (objConf.algo && objConf.masterKeyId) { + obj.kmsKeyInfo = await helpers.createKmsKey(log); + obj.kmsKey = objConf.arnPrefix ? obj.kmsKeyInfo.masterKeyArn : obj.kmsKeyInfo.masterKeyId; + } + objForCopy = bkt.objs[objConf.name]; + }); + + it(`should PutObject ${obj.name} overriding bucket SSE`, async () => + scenarios.tests.putObjectOverrideSSE({ objConf, obj }, { bktConf, bkt }, 'after')); + + // CopyObject scenarios + [ + { name: `${obj.name} into encrypted destination bucket`, forceBktSSE: true }, + { name: `${obj.name} into same bucket with object SSE config` }, + { name: `from encrypted source into ${obj.name} with object SSE config` }, + ].forEach(({ name, forceBktSSE }, index) => + it(`should CopyObject ${name}`, async () => + await scenarios.tests.copyObjectAndSSE( + { copyBkt, objForCopy, copyObj }, + { objConf, obj }, + { bktConf, bkt }, + { index, forceBktSSE }, + 'after', + )), + ); + + // after SSE migration implementation all mpu with sse are fixed + it('should encrypt MPU and put 2 encrypted parts', async () => { + const mpuKey = `${obj.name}-mpu`; + const mpu = await helpers.s3.createMultipartUpload( + helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey), + ); + const partsBody = [`${obj.body}-MPU1`, `${obj.body}-MPU2`]; + const newParts = []; + for (const [index, body] of partsBody.entries()) { + const part = await scenarios.tests.mpuUploadPart( + { + UploadId: mpu.UploadId, + Bucket: bkt.name, + Body: body, + Key: mpuKey, + PartNumber: index + 1, + }, + mpu, + objConf.algo || bktConf.algo, + 'after', + ); + newParts.push(part); + } + await scenarios.tests.mpuComplete( + { UploadId: mpu.UploadId, Bucket: bkt.name, Key: mpuKey }, + { existingParts: [], newParts }, + mpu, + objConf.algo || bktConf.algo, + 'after', + ); + const assertion = { + Bucket: bkt.name, + Key: mpuKey, + Body: `${obj.body}-MPU1${obj.body}-MPU2`, + }; + await scenarios.assertObjectSSE(assertion, { objConf, obj }, { bktConf, bkt }, {}, 'after'); + }); + + it('should encrypt MPU and copy an encrypted parts from encrypted bucket', async () => { + const mpuKey = `${obj.name}-mpucopy`; + const mpu = await helpers.s3.createMultipartUpload( + helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey), + ); + const part1 = await scenarios.tests.mpuUploadPartCopy( + { + UploadId: mpu.UploadId, + Bucket: bkt.name, + Key: mpuKey, + PartNumber: 1, + CopySource: `${copyBkt}/${copyObj}`, + }, + mpu, + objConf.algo || bktConf.algo, + 'after', + ); + const part2 = await scenarios.tests.mpuUploadPart( + { + UploadId: mpu.UploadId, + Bucket: bkt.name, + Body: `${obj.body}-MPU2`, + Key: mpuKey, + PartNumber: 2, + }, + mpu, + objConf.algo || bktConf.algo, + 'after', + ); + + await scenarios.tests.mpuComplete( + { UploadId: mpu.UploadId, Bucket: bkt.name, Key: mpuKey }, + { existingParts: [], newParts: [part1, part2] }, + mpu, + objConf.algo || bktConf.algo, + 'after', + ); + const assertion = { + Bucket: bkt.name, + Key: mpuKey, + Body: `BODY(copy)${obj.body}-MPU2`, + }; + await scenarios.assertObjectSSE(assertion, { objConf, obj }, { bktConf, bkt }, {}, 'after'); + }); + + it('should encrypt MPU and copy an encrypted range parts from encrypted bucket', async () => { + const mpuKey = `${obj.name}-mpucopy`; + const mpu = await helpers.s3.createMultipartUpload( + helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey), + ); + // source body is "BODY(copy)" + // [copy, BODY] + const sourceRanges = ['bytes=5-8', 'bytes=0-3']; + const newParts = []; + for (const [index, range] of sourceRanges.entries()) { + const part = await scenarios.tests.mpuUploadPartCopy( + { + UploadId: mpu.UploadId, + Bucket: bkt.name, + Key: mpuKey, + PartNumber: index + 1, + CopySource: `${copyBkt}/${copyObj}`, + CopySourceRange: range, + }, + mpu, + objConf.algo || bktConf.algo, + 'after', + ); + newParts.push(part); + } + + await scenarios.tests.mpuComplete( + { UploadId: mpu.UploadId, Bucket: bkt.name, Key: mpuKey }, + { existingParts: [], newParts }, + mpu, + objConf.algo || bktConf.algo, + 'after', + ); + const assertion = { + Bucket: bkt.name, + Key: mpuKey, + Body: 'copyBODY', + }; + await scenarios.assertObjectSSE(assertion, { objConf, obj }, { bktConf, bkt }, {}, 'after'); + }); + + it(`should PutObject versioned with SSE ${obj.name}`, async () => { + // ensure versioned bucket is empty + await helpers.bucketUtil.empty(bkt.vname); + let { Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname }); + // regularly count versioned objects + assert.strictEqual(Versions?.length, 0); + + const bodyBase = `BODY(${obj.name})-base`; + await helpers.putEncryptedObject(bkt.vname, obj.name, objConf, obj.kmsKey, bodyBase); + const baseAssertion = { Bucket: bkt.vname, Key: obj.name }; + await scenarios.assertObjectSSE( + { ...baseAssertion, Body: bodyBase }, + { objConf, obj }, + { bktConf, bkt }, + {}, + 'after', + ); + ({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname })); + assert.strictEqual(Versions.length, 1); + + await helpers.s3.putBucketVersioning({ + Bucket: bkt.vname, + VersioningConfiguration: { Status: 'Enabled' }, + }); + + const bodyV1 = `BODY(${obj.name})-v1`; + const v1 = await helpers.putEncryptedObject(bkt.vname, obj.name, objConf, obj.kmsKey, bodyV1); + const bodyV2 = `BODY(${obj.name})-v2`; + const v2 = await helpers.putEncryptedObject(bkt.vname, obj.name, objConf, obj.kmsKey, bodyV2); + ({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname })); + assert.strictEqual(Versions.length, 3); + + const current = await helpers.s3.headObject({ Bucket: bkt.vname, Key: obj.name }); + assert.strictEqual(current.VersionId, v2.VersionId); // ensure versioning as expected + ({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname })); + assert.strictEqual(Versions.length, 3); + + await scenarios.assertObjectSSE( + { ...baseAssertion, Body: bodyV2 }, + { objConf, obj }, + { bktConf, bkt }, + {}, + 'after', + ); // v2 + await scenarios.assertObjectSSE( + { ...baseAssertion, VersionId: 'null', Body: bodyBase }, + { objConf, obj }, + { bktConf, bkt }, + {}, + 'after', + ); + await scenarios.assertObjectSSE( + { ...baseAssertion, VersionId: v1.VersionId, Body: bodyV1 }, + { objConf, obj }, + { bktConf, bkt }, + {}, + 'after', + ); + await scenarios.assertObjectSSE( + { ...baseAssertion, VersionId: v2.VersionId, Body: bodyV2 }, + { objConf, obj }, + { bktConf, bkt }, + {}, + 'after', + ); + ({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname })); + assert.strictEqual(Versions.length, 3); + + await helpers.s3.putBucketVersioning({ + Bucket: bkt.vname, + VersioningConfiguration: { Status: 'Suspended' }, + }); + + // should be fine after version suspension + await scenarios.assertObjectSSE( + { ...baseAssertion, Body: bodyV2 }, + { objConf, obj }, + { bktConf, bkt }, + {}, + 'after', + ); // v2 + await scenarios.assertObjectSSE( + { ...baseAssertion, VersionId: 'null', Body: bodyBase }, + { objConf, obj }, + { bktConf, bkt }, + {}, + 'after', + ); + await scenarios.assertObjectSSE( + { ...baseAssertion, VersionId: v1.VersionId, Body: bodyV1 }, + { objConf, obj }, + { bktConf, bkt }, + {}, + 'after', + ); + await scenarios.assertObjectSSE( + { ...baseAssertion, VersionId: v2.VersionId, Body: bodyV2 }, + { objConf, obj }, + { bktConf, bkt }, + {}, + 'after', + ); + ({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname })); + assert.strictEqual(Versions.length, 3); + + // put a new null version + const bodyFinal = `BODY(${obj.name})-final`; + await helpers.putEncryptedObject(bkt.vname, obj.name, objConf, obj.kmsKey, bodyFinal); + await scenarios.assertObjectSSE( + { ...baseAssertion, Body: bodyFinal }, + { objConf, obj }, + { bktConf, bkt }, + {}, + 'after', + ); // null + await scenarios.assertObjectSSE( + { ...baseAssertion, Body: bodyFinal }, + { objConf, obj }, + { bktConf, bkt }, + 'null', + 'after', + ); + ({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname })); + assert.strictEqual(Versions.length, 3); + }); + }), + ); + }), + ); it('should encrypt MPU and copy parts from every buckets and objects matrice', async () => { await helpers.s3.putBucketEncryption({ @@ -369,8 +471,7 @@ describe('SSE KMS arnPrefix', () => { ServerSideEncryptionConfiguration: helpers.hydrateSSEConfig({ algo: 'AES256' }), }); const mpuKey = 'mpucopy'; - const mpu = await helpers.s3.createMultipartUpload( - helpers.putObjParams(mpuCopyBkt, mpuKey, {}, null)); + const mpu = await helpers.s3.createMultipartUpload(helpers.putObjParams(mpuCopyBkt, mpuKey, {}, null)); const copyPartArg = { UploadId: mpu.UploadId, Bucket: mpuCopyBkt, @@ -380,18 +481,20 @@ describe('SSE KMS arnPrefix', () => { const uploadPromises = scenarios.testCases.reduce((acc, bktConf, bktIdx) => { const bkt = bkts[bktConf.name]; - return acc.concat(scenarios.testCasesObj.map(async (objConf, objIdx) => { - const obj = bkt.objs[objConf.name]; + return acc.concat( + scenarios.testCasesObj.map(async (objConf, objIdx) => { + const obj = bkt.objs[objConf.name]; - const partNumber = bktIdx * scenarios.testCasesObj.length + objIdx + 1; - const res = await helpers.s3.uploadPartCopy({ - ...copyPartArg, - PartNumber: partNumber, - CopySource: `${bkt.name}/${obj.name}`, - }); + const partNumber = bktIdx * scenarios.testCasesObj.length + objIdx + 1; + const res = await helpers.s3.uploadPartCopy({ + ...copyPartArg, + PartNumber: partNumber, + CopySource: `${bkt.name}/${obj.name}`, + }); - return { partNumber, body: obj.body, res: res.CopyPartResult }; - })); + return { partNumber, body: obj.body, res: res.CopyPartResult }; + }), + ); }, []); const parts = await Promise.all(uploadPromises); @@ -409,8 +512,13 @@ describe('SSE KMS arnPrefix', () => { Key: mpuKey, Body: parts.reduce((acc, part) => `${acc}${part.body}`, ''), }; - await scenarios.assertObjectSSE(assertion, { objConf: {}, obj: {} }, - { bktConf: { algo: 'AES256' }, bkt: {} }, {}, 'after'); + await scenarios.assertObjectSSE( + assertion, + { objConf: {}, obj: {} }, + { bktConf: { algo: 'AES256' }, bkt: {} }, + {}, + 'after', + ); }); }); @@ -424,8 +532,11 @@ describe('ensure MPU use good SSE', () => { await helpers.s3.createBucket({ Bucket: mpuKmsBkt }); await helpers.s3.putBucketEncryption({ Bucket: mpuKmsBkt, - ServerSideEncryptionConfiguration: - helpers.hydrateSSEConfig({ algo: 'aws:kms', masterKeyId: kmsKeympuKmsBkt }) }); + ServerSideEncryptionConfiguration: helpers.hydrateSSEConfig({ + algo: 'aws:kms', + masterKeyId: kmsKeympuKmsBkt, + }), + }); }); after(async () => { @@ -435,7 +546,9 @@ describe('ensure MPU use good SSE', () => { it('mpu upload part should fail with sse header', async () => { const key = 'mpuKeyBadUpload'; const mpu = await helpers.s3.createMultipartUpload({ - Bucket: mpuKmsBkt, Key: key }); + Bucket: mpuKmsBkt, + Key: key, + }); const res = await promisify(makeRequest)({ method: 'PUT', hostname: helpers.s3.config.endpoint.hostname, @@ -465,21 +578,33 @@ describe('ensure MPU use good SSE', () => { const key = 'mpuKey'; const mpuKms = (await helpers.createKmsKey(log)).masterKeyArn; const mpu = await helpers.s3.createMultipartUpload({ - Bucket: mpuKmsBkt, Key: key, ServerSideEncryption: 'aws:kms', SSEKMSKeyId: mpuKms }); + Bucket: mpuKmsBkt, + Key: key, + ServerSideEncryption: 'aws:kms', + SSEKMSKeyId: mpuKms, + }); assert.strictEqual(mpu.ServerSideEncryption, 'aws:kms'); assert.strictEqual(mpu.SSEKMSKeyId, helpers.getKey(mpuKms)); - const part1 = await scenarios.tests.mpuUploadPart({ - UploadId: mpu.UploadId, - Bucket: mpuKmsBkt, - Body: 'Scality', - Key: key, - PartNumber: 1, - }, mpu, 'aws:kms', 'after'); + const part1 = await scenarios.tests.mpuUploadPart( + { + UploadId: mpu.UploadId, + Bucket: mpuKmsBkt, + Body: 'Scality', + Key: key, + PartNumber: 1, + }, + mpu, + 'aws:kms', + 'after', + ); await scenarios.tests.mpuComplete( { UploadId: mpu.UploadId, Bucket: mpuKmsBkt, Key: key }, { existingParts: [], newParts: [part1] }, - mpu, 'aws:kms', 'after'); + mpu, + 'aws:kms', + 'after', + ); const assertion = { Bucket: mpuKmsBkt, @@ -492,16 +617,12 @@ describe('ensure MPU use good SSE', () => { }; const bktForAssert = { bktConf: { algo: 'aws:kms', masterKeyId: true }, - bkt: { kmsKey: kmsKeympuKmsBkt, - kmsKeyInfo: { masterKeyId: kmsKeympuKmsBkt, masterKeyArn: kmsKeympuKmsBkt } }, + bkt: { + kmsKey: kmsKeympuKmsBkt, + kmsKeyInfo: { masterKeyId: kmsKeympuKmsBkt, masterKeyArn: kmsKeympuKmsBkt }, + }, }; - await scenarios.assertObjectSSE( - assertion, - objForAssert, - bktForAssert, - {}, - 'after', - ); + await scenarios.assertObjectSSE(assertion, objForAssert, bktForAssert, {}, 'after'); }); }); @@ -533,10 +654,11 @@ describe('KMS error', () => { */ const expectedLocalKms = { code: 'KMS.AccessDeniedException', - msg: () => new RegExp( - 'The ciphertext refers to a customer master key that does not exist, ' + - 'does not exist in this region, or you are not allowed to access\\.' - ), + msg: () => + new RegExp( + 'The ciphertext refers to a customer master key that does not exist, ' + + 'does not exist in this region, or you are not allowed to access\\.', + ), }; if (helpers.config.backends.kms === 'kmip') { expected = expectedKMIP; @@ -567,8 +689,7 @@ describe('KMS error', () => { Body: body, }); - mpuPlaintext = await helpers.s3.createMultipartUpload( - helpers.putObjParams(Bucket, 'mpuPlaintext', {}, null)); + mpuPlaintext = await helpers.s3.createMultipartUpload(helpers.putObjParams(Bucket, 'mpuPlaintext', {}, null)); ({ masterKeyId, masterKeyArn } = await helpers.createKmsKey(log)); @@ -578,7 +699,8 @@ describe('KMS error', () => { assert.strictEqual(obj.Body.toString(), body); mpuEncrypted = await helpers.s3.createMultipartUpload( - helpers.putObjParams(Bucket, 'mpuEncrypted', sseConfig, masterKeyArn)); + helpers.putObjParams(Bucket, 'mpuEncrypted', sseConfig, masterKeyArn), + ); // make key unavailable await helpers.destroyKmsKey(masterKeyArn, log); @@ -589,70 +711,89 @@ describe('KMS error', () => { if (masterKeyArn) { try { await helpers.destroyKmsKey(masterKeyArn, log); - } catch (e) { void e; } + } catch (e) { + void e; + } [masterKeyArn, masterKeyId] = [null, null]; } }); const testCases = [ { - action: 'putObject', kmsAction: 'Encrypt', - fct: async ({ masterKeyArn }) => - helpers.putEncryptedObject(Bucket, 'fail', sseConfig, masterKeyArn, body), + action: 'putObject', + kmsAction: 'Encrypt', + fct: async ({ masterKeyArn }) => helpers.putEncryptedObject(Bucket, 'fail', sseConfig, masterKeyArn, body), }, { - action: 'getObject', kmsAction: 'Decrypt', + action: 'getObject', + kmsAction: 'Decrypt', fct: async () => helpers.s3.getObject({ Bucket, Key }), }, { - action: 'copyObject', detail: ' when getting from source', kmsAction: 'Decrypt', - fct: async () => - helpers.s3.copyObject({ Bucket, Key: 'copy', CopySource: `${Bucket}/${Key}` }), + action: 'copyObject', + detail: ' when getting from source', + kmsAction: 'Decrypt', + fct: async () => helpers.s3.copyObject({ Bucket, Key: 'copy', CopySource: `${Bucket}/${Key}` }), }, { - action: 'copyObject', detail: ' when putting to destination', kmsAction: 'Encrypt', - fct: async ({ masterKeyArn }) => helpers.s3.copyObject({ - Bucket, - Key: 'copyencrypted', - CopySource: `${Bucket}/plaintext`, - ServerSideEncryption: 'aws:kms', - SSEKMSKeyId: masterKeyArn, - }), + action: 'copyObject', + detail: ' when putting to destination', + kmsAction: 'Encrypt', + fct: async ({ masterKeyArn }) => + helpers.s3.copyObject({ + Bucket, + Key: 'copyencrypted', + CopySource: `${Bucket}/plaintext`, + ServerSideEncryption: 'aws:kms', + SSEKMSKeyId: masterKeyArn, + }), }, { - action: 'createMPU', kmsAction: 'Encrypt', - fct: async ({ masterKeyArn }) => helpers.s3.createMultipartUpload( - helpers.putObjParams(Bucket, 'mpuKeyEncryptedFail', sseConfig, masterKeyArn)) , + action: 'createMPU', + kmsAction: 'Encrypt', + fct: async ({ masterKeyArn }) => + helpers.s3.createMultipartUpload( + helpers.putObjParams(Bucket, 'mpuKeyEncryptedFail', sseConfig, masterKeyArn), + ), }, { - action: 'mpu uploadPartCopy', detail: ' when getting from source', kmsAction: 'Decrypt', - fct: async ({ mpuPlaintext }) => helpers.s3.uploadPartCopy({ - UploadId: mpuPlaintext.UploadId, - Bucket, - Key: 'mpuPlaintext', - PartNumber: 1, - CopySource: `${Bucket}/${Key}`, - }), + action: 'mpu uploadPartCopy', + detail: ' when getting from source', + kmsAction: 'Decrypt', + fct: async ({ mpuPlaintext }) => + helpers.s3.uploadPartCopy({ + UploadId: mpuPlaintext.UploadId, + Bucket, + Key: 'mpuPlaintext', + PartNumber: 1, + CopySource: `${Bucket}/${Key}`, + }), }, { - action: 'mpu uploadPart', detail: ' when putting to destination', kmsAction: 'Encrypt', - fct: async ({ mpuEncrypted }) => helpers.s3.uploadPart({ - UploadId: mpuEncrypted.UploadId, - Bucket, - Key: 'mpuEncrypted', - PartNumber: 1, - Body: body, - }), + action: 'mpu uploadPart', + detail: ' when putting to destination', + kmsAction: 'Encrypt', + fct: async ({ mpuEncrypted }) => + helpers.s3.uploadPart({ + UploadId: mpuEncrypted.UploadId, + Bucket, + Key: 'mpuEncrypted', + PartNumber: 1, + Body: body, + }), }, { - action: 'mpu uploadPartCopy', detail: ' when putting to destination', kmsAction: 'Encrypt', - fct: async ({ mpuEncrypted }) => helpers.s3.uploadPartCopy({ - UploadId: mpuEncrypted.UploadId, - Bucket, - Key: 'mpuEncrypted', - PartNumber: 1, - CopySource: `${Bucket}/plaintext`, - }), + action: 'mpu uploadPartCopy', + detail: ' when putting to destination', + kmsAction: 'Encrypt', + fct: async ({ mpuEncrypted }) => + helpers.s3.uploadPartCopy({ + UploadId: mpuEncrypted.UploadId, + Bucket, + Key: 'mpuEncrypted', + PartNumber: 1, + CopySource: `${Bucket}/plaintext`, + }), }, ]; diff --git a/tests/functional/sse-kms-migration/beforeMigration.js b/tests/functional/sse-kms-migration/beforeMigration.js index 1a7a024761..135603e119 100644 --- a/tests/functional/sse-kms-migration/beforeMigration.js +++ b/tests/functional/sse-kms-migration/beforeMigration.js @@ -12,7 +12,11 @@ const scenarios = require('./scenarios'); // copy part of aws-node-sdk/test/object/encryptionHeaders.js and add more tests // Fix for before migration run to not add a prefix -Object.defineProperty(kms, 'arnPrefix', { get() { return ''; } }); +Object.defineProperty(kms, 'arnPrefix', { + get() { + return ''; + }, +}); describe('SSE KMS before migration', () => { /** Bucket to test CopyObject from and to */ @@ -36,9 +40,7 @@ describe('SSE KMS before migration', () => { if (bktConf.algo && bktConf.masterKeyId) { const key = crypto.randomBytes(32).toString('hex'); bkt.kmsKeyInfo = { masterKeyId: key, masterKeyArn: `${kms.arnPrefix}${key}` }; - bkt.kmsKey = bktConf.arnPrefix - ? bkt.kmsKeyInfo.masterKeyArn - : bkt.kmsKeyInfo.masterKeyId; + bkt.kmsKey = bktConf.arnPrefix ? bkt.kmsKeyInfo.masterKeyArn : bkt.kmsKeyInfo.masterKeyId; } await helpers.s3.createBucket({ Bucket: bkt.name }); await helpers.s3.createBucket({ Bucket: bkt.vname }); @@ -47,38 +49,44 @@ describe('SSE KMS before migration', () => { await helpers.s3.putBucketEncryption({ Bucket: bkt.name, ServerSideEncryptionConfiguration: helpers.hydrateSSEConfig({ - algo: bktConf.algo, masterKeyId: bkt.kmsKey }), + algo: bktConf.algo, + masterKeyId: bkt.kmsKey, + }), }); await helpers.s3.putBucketEncryption({ Bucket: bkt.vname, ServerSideEncryptionConfiguration: helpers.hydrateSSEConfig({ - algo: bktConf.algo, masterKeyId: bkt.kmsKey }), + algo: bktConf.algo, + masterKeyId: bkt.kmsKey, + }), }); } // Put an object for each SSE conf in each bucket - await Promise.all(scenarios.testCases.map(async objConf => { - const obj = { - name: `for-copy-enc-obj-${objConf.name}`, - kmsKeyInfo: null, - kmsKey: null, - body: `BODY(for-copy-enc-obj-${objConf.name})`, - }; - bkt.objs[objConf.name] = obj; - if (objConf.algo && objConf.masterKeyId) { - const key = crypto.randomBytes(32).toString('hex'); - obj.kmsKeyInfo = { masterKeyId: key, masterKeyArn: `${kms.arnPrefix}${key}` }; - obj.kmsKey = objConf.arnPrefix - ? obj.kmsKeyInfo.masterKeyArn - : obj.kmsKeyInfo.masterKeyId; - } - return await helpers.putEncryptedObject(bkt.name, obj.name, objConf, obj.kmsKey, obj.body); - })); + await Promise.all( + scenarios.testCases.map(async objConf => { + const obj = { + name: `for-copy-enc-obj-${objConf.name}`, + kmsKeyInfo: null, + kmsKey: null, + body: `BODY(for-copy-enc-obj-${objConf.name})`, + }; + bkt.objs[objConf.name] = obj; + if (objConf.algo && objConf.masterKeyId) { + const key = crypto.randomBytes(32).toString('hex'); + obj.kmsKeyInfo = { masterKeyId: key, masterKeyArn: `${kms.arnPrefix}${key}` }; + obj.kmsKey = objConf.arnPrefix ? obj.kmsKeyInfo.masterKeyArn : obj.kmsKeyInfo.masterKeyId; + } + return await helpers.putEncryptedObject(bkt.name, obj.name, objConf, obj.kmsKey, obj.body); + }), + ); }; before(async () => { - console.log('Run before migration', - { profile: helpers.credsProfile, accessKeyId: helpers.s3.config.credentials.accessKeyId }); + console.log('Run before migration', { + profile: helpers.credsProfile, + accessKeyId: helpers.s3.config.credentials.accessKeyId, + }); const allBuckets = ((await helpers.s3.listBuckets({})).Buckets || []).map(b => b.Name); console.log('List buckets:', allBuckets); await promisify(metadata.setup.bind(metadata))(); @@ -96,337 +104,457 @@ describe('SSE KMS before migration', () => { await Promise.all(scenarios.testCases.map(async bktConf => this.initBucket(bktConf))); }); - scenarios.testCases.forEach(bktConf => describe(`bucket enc-bkt-${bktConf.name}`, () => { - let bkt = bkts[bktConf.name]; - - before(() => { - bkt = bkts[bktConf.name]; - }); + scenarios.testCases.forEach(bktConf => + describe(`bucket enc-bkt-${bktConf.name}`, () => { + let bkt = bkts[bktConf.name]; - if (bktConf.deleteSSE) { - beforeEach(async () => scenarios.deleteBucketSSEBeforeEach(bkt.name, log)); - } - - if (!bktConf.algo) { - it('GetBucketEncryption should return ServerSideEncryptionConfigurationNotFoundError', - async () => await scenarios.tests.getBucketSSEError(bkt.name)); + before(() => { + bkt = bkts[bktConf.name]; + }); - if (!bktConf.deleteSSE) { - it('should have non mandatory SSE in bucket MD as test init put an object with AES256', - async () => await scenarios.tests.getBucketNonMandatorySSE(bkt.name, log, 'before')); + if (bktConf.deleteSSE) { + beforeEach(async () => scenarios.deleteBucketSSEBeforeEach(bkt.name, log)); } - } else { - it('GetBucketEncryption should return SSE with arnPrefix to key', - async () => await scenarios.tests.getBucketSSE(bkt.name, log, bktConf.algo, - bktConf.masterKeyId ? bkt.kmsKeyInfo.masterKeyArn : null, 'before')); - } - scenarios.testCasesObj.forEach(objConf => it(`should have pre uploaded object with SSE ${objConf.name}`, - async () => scenarios.tests.getPreUploadedObject(bkt.name, - { objConf, obj: bkt.objs[objConf.name] }, { bktConf, bkt }))); + if (!bktConf.algo) { + it('GetBucketEncryption should return ServerSideEncryptionConfigurationNotFoundError', async () => + await scenarios.tests.getBucketSSEError(bkt.name)); - scenarios.testCasesObj.forEach(objConf => describe(`object enc-obj-${objConf.name}`, () => { - const obj = { - name: `enc-obj-${objConf.name}`, - kmsKeyInfo: null, - kmsKey: null, - body: `BODY(enc-obj-${objConf.name})`, - }; - /** to be used as source of copy */ - let objForCopy; - - before(async () => { - if (objConf.algo && objConf.masterKeyId) { - const key = crypto.randomBytes(32).toString('hex'); - obj.kmsKeyInfo = { masterKeyId: key, masterKeyArn: `${kms.arnPrefix}${key}` }; - obj.kmsKey = objConf.arnPrefix - ? obj.kmsKeyInfo.masterKeyArn - : obj.kmsKeyInfo.masterKeyId; + if (!bktConf.deleteSSE) { + it('should have non mandatory SSE in bucket MD as test init put an object with AES256', async () => + await scenarios.tests.getBucketNonMandatorySSE(bkt.name, log, 'before')); } - objForCopy = bkt.objs[objConf.name]; - }); - - it(`should PutObject ${obj.name} overriding bucket SSE`, - async () => scenarios.tests.putObjectOverrideSSE({ objConf, obj }, { bktConf, bkt })); - - // CopyObject scenarios - [ - { name: `${obj.name} into encrypted destination bucket`, forceBktSSE: true }, - { name: `${obj.name} into same bucket with object SSE config` }, - { name: `from encrypted source into ${obj.name} with object SSE config` }, - ].forEach(({ name, forceBktSSE }, index) => - it(`should CopyObject ${name}`, async () => - await scenarios.tests.copyObjectAndSSE( - { copyBkt, objForCopy, copyObj }, - { objConf, obj }, - { bktConf, bkt }, - { index, forceBktSSE }, + } else { + it('GetBucketEncryption should return SSE with arnPrefix to key', async () => + await scenarios.tests.getBucketSSE( + bkt.name, + log, + bktConf.algo, + bktConf.masterKeyId ? bkt.kmsKeyInfo.masterKeyArn : null, 'before', - ))); - - // S3C-9996 The SSE was bugged with MPU, where the completion takes only the masterKeyId from bucket - // Fixed at the same time as migration, some scenario can pass only in newer version above migration - const optionalSkip = objConf.algo || bktConf.masterKeyId || (!bktConf.algo && !bktConf.deleteSSE) - ? it.skip - : it; - optionalSkip('should encrypt MPU and put 2 encrypted parts', async () => { - const mpuKey = `${obj.name}-mpu`; - const mpu = await helpers.s3.createMultipartUpload( - helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey)); - const partsBody = [`${obj.body}-MPU1`, `${obj.body}-MPU2`]; - const newParts = []; - for (const [index, body] of partsBody.entries()) { - const part = await scenarios.tests.mpuUploadPart({ - UploadId: mpu.UploadId, - Bucket: bkt.name, - Body: body, - Key: mpuKey, - PartNumber: index + 1, - }, mpu, objConf.algo || bktConf.algo, 'before'); - newParts.push(part); - } - await scenarios.tests.mpuComplete( - { UploadId: mpu.UploadId, Bucket: bkt.name, Key: mpuKey }, - { existingParts: [], newParts }, - mpu, objConf.algo || bktConf.algo, 'before'); - const assertion = { - Bucket: bkt.name, - Key: mpuKey, - Body: `${obj.body}-MPU1${obj.body}-MPU2`, - }; - await scenarios.assertObjectSSE(assertion, { objConf, obj }, { bktConf, bkt }); - }); - - optionalSkip('should encrypt MPU and copy an encrypted parts from encrypted bucket', async () => { - const mpuKey = `${obj.name}-mpucopy`; - const mpu = await helpers.s3.createMultipartUpload( - helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey)); - const part1 = await scenarios.tests.mpuUploadPartCopy({ - UploadId: mpu.UploadId, - Bucket: bkt.name, - Key: mpuKey, - PartNumber: 1, - CopySource: `${copyBkt}/${copyObj}`, - }, mpu, objConf.algo || bktConf.algo, 'before'); - const part2 = await scenarios.tests.mpuUploadPart({ - UploadId: mpu.UploadId, - Bucket: bkt.name, - Body: `${obj.body}-MPU2`, - Key: mpuKey, - PartNumber: 2, - }, mpu, objConf.algo || bktConf.algo, 'before'); - - await scenarios.tests.mpuComplete( - { UploadId: mpu.UploadId, Bucket: bkt.name, Key: mpuKey }, - { existingParts: [], newParts: [part1, part2] }, - mpu, objConf.algo || bktConf.algo, 'before'); - const assertion = { - Bucket: bkt.name, - Key: mpuKey, - Body: `BODY(copy)${obj.body}-MPU2`, - }; - await scenarios.assertObjectSSE(assertion, { objConf, obj }, { bktConf, bkt }); - }); - - optionalSkip('should encrypt MPU and copy an encrypted range parts from encrypted bucket', async () => { - const mpuKey = `${obj.name}-mpucopyrange`; - const mpu = await helpers.s3.createMultipartUpload( - helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey)); - // source body is "BODY(copy)" - // [copy, BODY] - const sourceRanges = ['bytes=5-8', 'bytes=0-3']; - const newParts = []; - for (const [index, range] of sourceRanges.entries()) { - const part = await scenarios.tests.mpuUploadPartCopy({ - UploadId: mpu.UploadId, - Bucket: bkt.name, - Key: mpuKey, - PartNumber: index + 1, - CopySource: `${copyBkt}/${copyObj}`, - CopySourceRange: range, - }, mpu, objConf.algo || bktConf.algo, 'before'); - newParts.push(part); - } - - await scenarios.tests.mpuComplete( - { UploadId: mpu.UploadId, Bucket: bkt.name, Key: mpuKey }, - { existingParts: [], newParts }, - mpu, objConf.algo || bktConf.algo, 'before'); - const assertion = { - Bucket: bkt.name, - Key: mpuKey, - Body: 'copyBODY', - }; - await scenarios.assertObjectSSE(assertion, { objConf, obj }, { bktConf, bkt }); - }); - - optionalSkip('should prepare empty encrypted MPU without completion', async () => { - const mpuKey = `${obj.name}-migration-mpu-empty`; - await helpers.s3.createMultipartUpload( - helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey)); - }); - - optionalSkip('should prepare encrypted MPU and put 2 encrypted parts without completion', async () => { - const mpuKey = `${obj.name}-migration-mpu`; - const mpu = await helpers.s3.createMultipartUpload( - helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey)); - const partsBody = [`${obj.body}-MPU1`, `${obj.body}-MPU2`]; - for (const [index, body] of partsBody.entries()) { - await scenarios.tests.mpuUploadPart({ - UploadId: mpu.UploadId, - Bucket: bkt.name, - Body: body, - Key: mpuKey, - PartNumber: index + 1, - }, mpu, objConf.algo || bktConf.algo, 'before'); - } - }); - - optionalSkip('should prepare encrypted MPU and copy an encrypted parts ' + - 'from encrypted bucket without completion', async () => { - const mpuKey = `${obj.name}-migration-mpucopy`; - const mpu = await helpers.s3.createMultipartUpload( - helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey)); - await scenarios.tests.mpuUploadPartCopy({ - UploadId: mpu.UploadId, - Bucket: bkt.name, - Key: mpuKey, - PartNumber: 1, - CopySource: `${copyBkt}/${copyObj}`, - }, mpu, objConf.algo || bktConf.algo, 'before'); - await scenarios.tests.mpuUploadPart({ - UploadId: mpu.UploadId, - Bucket: bkt.name, - Body: `${obj.body}-MPU2`, - Key: mpuKey, - PartNumber: 2, - }, mpu, objConf.algo || bktConf.algo, 'before'); - }); - - optionalSkip('should prepare encrypted MPU and copy an encrypted range parts ' + - 'from encrypted bucket without completion', async () => { - const mpuKey = `${obj.name}-migration-mpucopyrange`; - const mpu = await helpers.s3.createMultipartUpload( - helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey)); - // source body is "BODY(copy)" - // [copy, BODY] - const sourceRanges = ['bytes=5-8', 'bytes=0-3']; - for (const [index, range] of sourceRanges.entries()) { - await scenarios.tests.mpuUploadPartCopy({ - UploadId: mpu.UploadId, - Bucket: bkt.name, - Key: mpuKey, - PartNumber: index + 1, - CopySource: `${copyBkt}/${copyObj}`, - CopySourceRange: range, - }, mpu, objConf.algo || bktConf.algo, 'before'); - } - }); + )); + } - it(`should PutObject versioned with SSE ${obj.name}`, async () => { - // ensure versioned bucket is empty - await helpers.bucketUtil.empty(bkt.vname); - let { Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname }) || { Versions: [] }; - assert.strictEqual(Versions.length, 0); - - const bodyBase = `BODY(${obj.name})-base`; - await helpers.putEncryptedObject(bkt.vname, obj.name, objConf, obj.kmsKey, bodyBase); - const baseAssertion = { Bucket: bkt.vname, Key: obj.name }; - await scenarios.assertObjectSSE( - { ...baseAssertion, Body: bodyBase }, - { objConf, obj }, { bktConf, bkt }); - ({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname })); - assert.strictEqual(Versions.length, 1); - - await helpers.s3.putBucketVersioning({ Bucket: bkt.vname, - VersioningConfiguration: { Status: 'Enabled' }, - }); - - const bodyV1 = `BODY(${obj.name})-v1`; - const v1 = await helpers.putEncryptedObject(bkt.vname, obj.name, objConf, obj.kmsKey, bodyV1); - const bodyV2 = `BODY(${obj.name})-v2`; - const v2 = await helpers.putEncryptedObject(bkt.vname, obj.name, objConf, obj.kmsKey, bodyV2); - ({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname })); - assert.strictEqual(Versions.length, 3); - - const current = await helpers.s3.headObject({ Bucket: bkt.vname, Key: obj.name }); - assert.strictEqual(current.VersionId, v2.VersionId); // ensure versioning as expected - ({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname })); - assert.strictEqual(Versions.length, 3); - - await scenarios.assertObjectSSE( - { ...baseAssertion, Body: bodyV2 }, { objConf, obj }, { bktConf, bkt }); // v2 - await scenarios.assertObjectSSE( - { ...baseAssertion, VersionId: 'null', Body: bodyBase }, { objConf, obj }, { bktConf, bkt }); - await scenarios.assertObjectSSE( - { ...baseAssertion, VersionId: v1.VersionId, Body: bodyV1 }, { objConf, obj }, { bktConf, bkt }); - await scenarios.assertObjectSSE( - { ...baseAssertion, VersionId: v2.VersionId, Body: bodyV2 }, { objConf, obj }, { bktConf, bkt }); - ({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname })); - assert.strictEqual(Versions.length, 3); - - await helpers.s3.putBucketVersioning({ Bucket: bkt.vname, - VersioningConfiguration: { Status: 'Suspended' }, - }); - - // should be fine after version suspension - await scenarios.assertObjectSSE( - { ...baseAssertion, Body: bodyV2 }, { objConf, obj }, { bktConf, bkt }); // v2 - await scenarios.assertObjectSSE( - { ...baseAssertion, VersionId: 'null', Body: bodyBase }, { objConf, obj }, { bktConf, bkt }); - await scenarios.assertObjectSSE( - { ...baseAssertion, VersionId: v1.VersionId, Body: bodyV1 }, { objConf, obj }, { bktConf, bkt }); - await scenarios.assertObjectSSE( - { ...baseAssertion, VersionId: v2.VersionId, Body: bodyV2 }, { objConf, obj }, { bktConf, bkt }); - ({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname })); - assert.strictEqual(Versions.length, 3); - - // put a new null version - const bodyFinal = `BODY(${obj.name})-final`; - await helpers.putEncryptedObject(bkt.vname, obj.name, objConf, obj.kmsKey, bodyFinal); - await scenarios.assertObjectSSE( - { ...baseAssertion, Body: bodyFinal }, { objConf, obj }, { bktConf, bkt }); // null - await scenarios.assertObjectSSE( - { ...baseAssertion, Body: bodyFinal }, { objConf, obj }, { bktConf, bkt }, 'null'); - ({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname })); - assert.strictEqual(Versions.length, 3); + scenarios.testCasesObj.forEach(objConf => + it(`should have pre uploaded object with SSE ${objConf.name}`, async () => + scenarios.tests.getPreUploadedObject( + bkt.name, + { objConf, obj: bkt.objs[objConf.name] }, + { bktConf, bkt }, + )), + ); + + scenarios.testCasesObj.forEach(objConf => + describe(`object enc-obj-${objConf.name}`, () => { + const obj = { + name: `enc-obj-${objConf.name}`, + kmsKeyInfo: null, + kmsKey: null, + body: `BODY(enc-obj-${objConf.name})`, + }; + /** to be used as source of copy */ + let objForCopy; + + before(async () => { + if (objConf.algo && objConf.masterKeyId) { + const key = crypto.randomBytes(32).toString('hex'); + obj.kmsKeyInfo = { masterKeyId: key, masterKeyArn: `${kms.arnPrefix}${key}` }; + obj.kmsKey = objConf.arnPrefix ? obj.kmsKeyInfo.masterKeyArn : obj.kmsKeyInfo.masterKeyId; + } + objForCopy = bkt.objs[objConf.name]; + }); + + it(`should PutObject ${obj.name} overriding bucket SSE`, async () => + scenarios.tests.putObjectOverrideSSE({ objConf, obj }, { bktConf, bkt })); + + // CopyObject scenarios + [ + { name: `${obj.name} into encrypted destination bucket`, forceBktSSE: true }, + { name: `${obj.name} into same bucket with object SSE config` }, + { name: `from encrypted source into ${obj.name} with object SSE config` }, + ].forEach(({ name, forceBktSSE }, index) => + it(`should CopyObject ${name}`, async () => + await scenarios.tests.copyObjectAndSSE( + { copyBkt, objForCopy, copyObj }, + { objConf, obj }, + { bktConf, bkt }, + { index, forceBktSSE }, + 'before', + )), + ); + + // S3C-9996 The SSE was bugged with MPU, where the completion takes only the masterKeyId from bucket + // Fixed at the same time as migration, some scenario can pass only in newer version above migration + const optionalSkip = + objConf.algo || bktConf.masterKeyId || (!bktConf.algo && !bktConf.deleteSSE) ? it.skip : it; + optionalSkip('should encrypt MPU and put 2 encrypted parts', async () => { + const mpuKey = `${obj.name}-mpu`; + const mpu = await helpers.s3.createMultipartUpload( + helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey), + ); + const partsBody = [`${obj.body}-MPU1`, `${obj.body}-MPU2`]; + const newParts = []; + for (const [index, body] of partsBody.entries()) { + const part = await scenarios.tests.mpuUploadPart( + { + UploadId: mpu.UploadId, + Bucket: bkt.name, + Body: body, + Key: mpuKey, + PartNumber: index + 1, + }, + mpu, + objConf.algo || bktConf.algo, + 'before', + ); + newParts.push(part); + } + await scenarios.tests.mpuComplete( + { UploadId: mpu.UploadId, Bucket: bkt.name, Key: mpuKey }, + { existingParts: [], newParts }, + mpu, + objConf.algo || bktConf.algo, + 'before', + ); + const assertion = { + Bucket: bkt.name, + Key: mpuKey, + Body: `${obj.body}-MPU1${obj.body}-MPU2`, + }; + await scenarios.assertObjectSSE(assertion, { objConf, obj }, { bktConf, bkt }); + }); + + optionalSkip('should encrypt MPU and copy an encrypted parts from encrypted bucket', async () => { + const mpuKey = `${obj.name}-mpucopy`; + const mpu = await helpers.s3.createMultipartUpload( + helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey), + ); + const part1 = await scenarios.tests.mpuUploadPartCopy( + { + UploadId: mpu.UploadId, + Bucket: bkt.name, + Key: mpuKey, + PartNumber: 1, + CopySource: `${copyBkt}/${copyObj}`, + }, + mpu, + objConf.algo || bktConf.algo, + 'before', + ); + const part2 = await scenarios.tests.mpuUploadPart( + { + UploadId: mpu.UploadId, + Bucket: bkt.name, + Body: `${obj.body}-MPU2`, + Key: mpuKey, + PartNumber: 2, + }, + mpu, + objConf.algo || bktConf.algo, + 'before', + ); + + await scenarios.tests.mpuComplete( + { UploadId: mpu.UploadId, Bucket: bkt.name, Key: mpuKey }, + { existingParts: [], newParts: [part1, part2] }, + mpu, + objConf.algo || bktConf.algo, + 'before', + ); + const assertion = { + Bucket: bkt.name, + Key: mpuKey, + Body: `BODY(copy)${obj.body}-MPU2`, + }; + await scenarios.assertObjectSSE(assertion, { objConf, obj }, { bktConf, bkt }); + }); + + optionalSkip( + 'should encrypt MPU and copy an encrypted range parts from encrypted bucket', + async () => { + const mpuKey = `${obj.name}-mpucopyrange`; + const mpu = await helpers.s3.createMultipartUpload( + helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey), + ); + // source body is "BODY(copy)" + // [copy, BODY] + const sourceRanges = ['bytes=5-8', 'bytes=0-3']; + const newParts = []; + for (const [index, range] of sourceRanges.entries()) { + const part = await scenarios.tests.mpuUploadPartCopy( + { + UploadId: mpu.UploadId, + Bucket: bkt.name, + Key: mpuKey, + PartNumber: index + 1, + CopySource: `${copyBkt}/${copyObj}`, + CopySourceRange: range, + }, + mpu, + objConf.algo || bktConf.algo, + 'before', + ); + newParts.push(part); + } + + await scenarios.tests.mpuComplete( + { UploadId: mpu.UploadId, Bucket: bkt.name, Key: mpuKey }, + { existingParts: [], newParts }, + mpu, + objConf.algo || bktConf.algo, + 'before', + ); + const assertion = { + Bucket: bkt.name, + Key: mpuKey, + Body: 'copyBODY', + }; + await scenarios.assertObjectSSE(assertion, { objConf, obj }, { bktConf, bkt }); + }, + ); + + optionalSkip('should prepare empty encrypted MPU without completion', async () => { + const mpuKey = `${obj.name}-migration-mpu-empty`; + await helpers.s3.createMultipartUpload( + helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey), + ); + }); + + optionalSkip( + 'should prepare encrypted MPU and put 2 encrypted parts without completion', + async () => { + const mpuKey = `${obj.name}-migration-mpu`; + const mpu = await helpers.s3.createMultipartUpload( + helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey), + ); + const partsBody = [`${obj.body}-MPU1`, `${obj.body}-MPU2`]; + for (const [index, body] of partsBody.entries()) { + await scenarios.tests.mpuUploadPart( + { + UploadId: mpu.UploadId, + Bucket: bkt.name, + Body: body, + Key: mpuKey, + PartNumber: index + 1, + }, + mpu, + objConf.algo || bktConf.algo, + 'before', + ); + } + }, + ); + + optionalSkip( + 'should prepare encrypted MPU and copy an encrypted parts ' + + 'from encrypted bucket without completion', + async () => { + const mpuKey = `${obj.name}-migration-mpucopy`; + const mpu = await helpers.s3.createMultipartUpload( + helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey), + ); + await scenarios.tests.mpuUploadPartCopy( + { + UploadId: mpu.UploadId, + Bucket: bkt.name, + Key: mpuKey, + PartNumber: 1, + CopySource: `${copyBkt}/${copyObj}`, + }, + mpu, + objConf.algo || bktConf.algo, + 'before', + ); + await scenarios.tests.mpuUploadPart( + { + UploadId: mpu.UploadId, + Bucket: bkt.name, + Body: `${obj.body}-MPU2`, + Key: mpuKey, + PartNumber: 2, + }, + mpu, + objConf.algo || bktConf.algo, + 'before', + ); + }, + ); + + optionalSkip( + 'should prepare encrypted MPU and copy an encrypted range parts ' + + 'from encrypted bucket without completion', + async () => { + const mpuKey = `${obj.name}-migration-mpucopyrange`; + const mpu = await helpers.s3.createMultipartUpload( + helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey), + ); + // source body is "BODY(copy)" + // [copy, BODY] + const sourceRanges = ['bytes=5-8', 'bytes=0-3']; + for (const [index, range] of sourceRanges.entries()) { + await scenarios.tests.mpuUploadPartCopy( + { + UploadId: mpu.UploadId, + Bucket: bkt.name, + Key: mpuKey, + PartNumber: index + 1, + CopySource: `${copyBkt}/${copyObj}`, + CopySourceRange: range, + }, + mpu, + objConf.algo || bktConf.algo, + 'before', + ); + } + }, + ); + + it(`should PutObject versioned with SSE ${obj.name}`, async () => { + // ensure versioned bucket is empty + await helpers.bucketUtil.empty(bkt.vname); + let { Versions } = (await helpers.s3.listObjectVersions({ Bucket: bkt.vname })) || { + Versions: [], + }; + assert.strictEqual(Versions.length, 0); + + const bodyBase = `BODY(${obj.name})-base`; + await helpers.putEncryptedObject(bkt.vname, obj.name, objConf, obj.kmsKey, bodyBase); + const baseAssertion = { Bucket: bkt.vname, Key: obj.name }; + await scenarios.assertObjectSSE( + { ...baseAssertion, Body: bodyBase }, + { objConf, obj }, + { bktConf, bkt }, + ); + ({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname })); + assert.strictEqual(Versions.length, 1); + + await helpers.s3.putBucketVersioning({ + Bucket: bkt.vname, + VersioningConfiguration: { Status: 'Enabled' }, + }); + + const bodyV1 = `BODY(${obj.name})-v1`; + const v1 = await helpers.putEncryptedObject(bkt.vname, obj.name, objConf, obj.kmsKey, bodyV1); + const bodyV2 = `BODY(${obj.name})-v2`; + const v2 = await helpers.putEncryptedObject(bkt.vname, obj.name, objConf, obj.kmsKey, bodyV2); + ({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname })); + assert.strictEqual(Versions.length, 3); + + const current = await helpers.s3.headObject({ Bucket: bkt.vname, Key: obj.name }); + assert.strictEqual(current.VersionId, v2.VersionId); // ensure versioning as expected + ({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname })); + assert.strictEqual(Versions.length, 3); + + await scenarios.assertObjectSSE( + { ...baseAssertion, Body: bodyV2 }, + { objConf, obj }, + { bktConf, bkt }, + ); // v2 + await scenarios.assertObjectSSE( + { ...baseAssertion, VersionId: 'null', Body: bodyBase }, + { objConf, obj }, + { bktConf, bkt }, + ); + await scenarios.assertObjectSSE( + { ...baseAssertion, VersionId: v1.VersionId, Body: bodyV1 }, + { objConf, obj }, + { bktConf, bkt }, + ); + await scenarios.assertObjectSSE( + { ...baseAssertion, VersionId: v2.VersionId, Body: bodyV2 }, + { objConf, obj }, + { bktConf, bkt }, + ); + ({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname })); + assert.strictEqual(Versions.length, 3); + + await helpers.s3.putBucketVersioning({ + Bucket: bkt.vname, + VersioningConfiguration: { Status: 'Suspended' }, + }); + + // should be fine after version suspension + await scenarios.assertObjectSSE( + { ...baseAssertion, Body: bodyV2 }, + { objConf, obj }, + { bktConf, bkt }, + ); // v2 + await scenarios.assertObjectSSE( + { ...baseAssertion, VersionId: 'null', Body: bodyBase }, + { objConf, obj }, + { bktConf, bkt }, + ); + await scenarios.assertObjectSSE( + { ...baseAssertion, VersionId: v1.VersionId, Body: bodyV1 }, + { objConf, obj }, + { bktConf, bkt }, + ); + await scenarios.assertObjectSSE( + { ...baseAssertion, VersionId: v2.VersionId, Body: bodyV2 }, + { objConf, obj }, + { bktConf, bkt }, + ); + ({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname })); + assert.strictEqual(Versions.length, 3); + + // put a new null version + const bodyFinal = `BODY(${obj.name})-final`; + await helpers.putEncryptedObject(bkt.vname, obj.name, objConf, obj.kmsKey, bodyFinal); + await scenarios.assertObjectSSE( + { ...baseAssertion, Body: bodyFinal }, + { objConf, obj }, + { bktConf, bkt }, + ); // null + await scenarios.assertObjectSSE( + { ...baseAssertion, Body: bodyFinal }, + { objConf, obj }, + { bktConf, bkt }, + 'null', + ); + ({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname })); + assert.strictEqual(Versions.length, 3); + }); + }), + ); + }), + ); + + it( + 'should prepare encrypted MPU and copy parts from ' + 'every buckets and objects matrice without completion', + async () => { + await helpers.s3.putBucketEncryption({ + Bucket: mpuCopyBkt, + // AES256 because input key is broken for now + ServerSideEncryptionConfiguration: helpers.hydrateSSEConfig({ algo: 'AES256' }), }); - })); - })); - - it('should prepare encrypted MPU and copy parts from ' + - 'every buckets and objects matrice without completion', async () => { - await helpers.s3.putBucketEncryption({ - Bucket: mpuCopyBkt, - // AES256 because input key is broken for now - ServerSideEncryptionConfiguration: helpers.hydrateSSEConfig({ algo: 'AES256' }), - }); - const mpuKey = 'mpucopy'; - const mpu = await helpers.s3.createMultipartUpload( - helpers.putObjParams(mpuCopyBkt, mpuKey, {}, null)); - const copyPartArg = { - UploadId: mpu.UploadId, - Bucket: mpuCopyBkt, - Key: mpuKey, - }; - // For each test Case bucket and object copy a part - const uploadPromises = scenarios.testCases.reduce((acc, bktConf, bktIdx) => { - const bkt = bkts[bktConf.name]; - - return acc.concat(scenarios.testCasesObj.map(async (objConf, objIdx) => { - const obj = bkt.objs[objConf.name]; - - const partNumber = bktIdx * scenarios.testCasesObj.length + objIdx + 1; - const res = await helpers.s3.uploadPartCopy({ - ...copyPartArg, - PartNumber: partNumber, - CopySource: `${bkt.name}/${obj.name}`, - }); - - return { partNumber, body: obj.body, res: res.CopyPartResult }; - })); - }, []); - - await Promise.all(uploadPromises); - }); + const mpuKey = 'mpucopy'; + const mpu = await helpers.s3.createMultipartUpload(helpers.putObjParams(mpuCopyBkt, mpuKey, {}, null)); + const copyPartArg = { + UploadId: mpu.UploadId, + Bucket: mpuCopyBkt, + Key: mpuKey, + }; + // For each test Case bucket and object copy a part + const uploadPromises = scenarios.testCases.reduce((acc, bktConf, bktIdx) => { + const bkt = bkts[bktConf.name]; + + return acc.concat( + scenarios.testCasesObj.map(async (objConf, objIdx) => { + const obj = bkt.objs[objConf.name]; + + const partNumber = bktIdx * scenarios.testCasesObj.length + objIdx + 1; + const res = await helpers.s3.uploadPartCopy({ + ...copyPartArg, + PartNumber: partNumber, + CopySource: `${bkt.name}/${obj.name}`, + }); + + return { partNumber, body: obj.body, res: res.CopyPartResult }; + }), + ); + }, []); + + await Promise.all(uploadPromises); + }, + ); }); diff --git a/tests/functional/sse-kms-migration/cleanup.js b/tests/functional/sse-kms-migration/cleanup.js index d1e5cb32e2..5be7b64b87 100644 --- a/tests/functional/sse-kms-migration/cleanup.js +++ b/tests/functional/sse-kms-migration/cleanup.js @@ -16,8 +16,10 @@ describe('SSE KMS Cleanup', () => { const mpuCopyBkt = 'enc-bkt-mpu-copy'; it('Empty and delete buckets for SSE KMS Migration', async () => { - console.log('Run cleanup', - { profile: helpers.credsProfile, accessKeyId: helpers.s3.config.credentials.accessKeyId }); + console.log('Run cleanup', { + profile: helpers.credsProfile, + accessKeyId: helpers.s3.config.credentials.accessKeyId, + }); const allBuckets = ((await helpers.s3.listBuckets()).Buckets || []).map(b => b.Name); console.log('List buckets:', allBuckets); @@ -26,10 +28,14 @@ describe('SSE KMS Cleanup', () => { try { await cleanup(copyBkt); await cleanup(mpuCopyBkt); - await Promise.all(scenarios.testCases.map(async bktConf => { - await cleanup(`enc-bkt-${bktConf.name}`); - return await cleanup(`versioned-enc-bkt-${bktConf.name}`); - })); - } catch (e) { void e; } + await Promise.all( + scenarios.testCases.map(async bktConf => { + await cleanup(`enc-bkt-${bktConf.name}`); + return await cleanup(`versioned-enc-bkt-${bktConf.name}`); + }), + ); + } catch (e) { + void e; + } }); }); diff --git a/tests/functional/sse-kms-migration/configs/aws.json b/tests/functional/sse-kms-migration/configs/aws.json index c593f310f5..913e031c99 100644 --- a/tests/functional/sse-kms-migration/configs/aws.json +++ b/tests/functional/sse-kms-migration/configs/aws.json @@ -1,9 +1,8 @@ { - "kmsAWS": { "noAwsArn": true, "providerName": "local", - "region": "us-east-1", + "region": "us-east-1", "endpoint": "http://0:8080", "ak": "456", "sk": "123" diff --git a/tests/functional/sse-kms-migration/configs/base.json b/tests/functional/sse-kms-migration/configs/base.json index 841aa341c7..9db4bea239 100644 --- a/tests/functional/sse-kms-migration/configs/base.json +++ b/tests/functional/sse-kms-migration/configs/base.json @@ -12,28 +12,33 @@ "127.0.0.2": "us-east-1", "s3.amazonaws.com": "us-east-1" }, - "websiteEndpoints": ["s3-website-us-east-1.amazonaws.com", - "s3-website.us-east-2.amazonaws.com", - "s3-website-us-west-1.amazonaws.com", - "s3-website-us-west-2.amazonaws.com", - "s3-website.ap-south-1.amazonaws.com", - "s3-website.ap-northeast-2.amazonaws.com", - "s3-website-ap-southeast-1.amazonaws.com", - "s3-website-ap-southeast-2.amazonaws.com", - "s3-website-ap-northeast-1.amazonaws.com", - "s3-website.eu-central-1.amazonaws.com", - "s3-website-eu-west-1.amazonaws.com", - "s3-website-sa-east-1.amazonaws.com", - "s3-website.localhost", - "s3-website.scality.test"], - "replicationEndpoints": [{ - "site": "zenko", - "servers": ["127.0.0.1:8000"], - "default": true - }, { - "site": "us-east-2", - "type": "aws_s3" - }], + "websiteEndpoints": [ + "s3-website-us-east-1.amazonaws.com", + "s3-website.us-east-2.amazonaws.com", + "s3-website-us-west-1.amazonaws.com", + "s3-website-us-west-2.amazonaws.com", + "s3-website.ap-south-1.amazonaws.com", + "s3-website.ap-northeast-2.amazonaws.com", + "s3-website-ap-southeast-1.amazonaws.com", + "s3-website-ap-southeast-2.amazonaws.com", + "s3-website-ap-northeast-1.amazonaws.com", + "s3-website.eu-central-1.amazonaws.com", + "s3-website-eu-west-1.amazonaws.com", + "s3-website-sa-east-1.amazonaws.com", + "s3-website.localhost", + "s3-website.scality.test" + ], + "replicationEndpoints": [ + { + "site": "zenko", + "servers": ["127.0.0.1:8000"], + "default": true + }, + { + "site": "us-east-2", + "type": "aws_s3" + } + ], "cdmi": { "host": "localhost", "port": 81, diff --git a/tests/functional/sse-kms-migration/configs/kmip-cluster.json b/tests/functional/sse-kms-migration/configs/kmip-cluster.json index 70ffcaf1b5..0723ab7260 100644 --- a/tests/functional/sse-kms-migration/configs/kmip-cluster.json +++ b/tests/functional/sse-kms-migration/configs/kmip-cluster.json @@ -4,7 +4,7 @@ "client": { "compoundCreateActivate": false }, - "transport": [ + "transport": [ { "pipelineDepth": 8, "tls": { diff --git a/tests/functional/sse-kms-migration/helpers.js b/tests/functional/sse-kms-migration/helpers.js index 456cade69d..c2e24bc1e2 100644 --- a/tests/functional/sse-kms-migration/helpers.js +++ b/tests/functional/sse-kms-migration/helpers.js @@ -1,5 +1,6 @@ const getConfig = require('../aws-node-sdk/test/support/config'); -const { S3Client, +const { + S3Client, CreateBucketCommand, DeleteBucketCommand, PutBucketEncryptionCommand, @@ -52,7 +53,7 @@ const httpsAgent = new HttpsAgent({ keepAliveMsecs: 30000, maxSockets: 50, maxFreeSockets: 10, - timeout: 120000, + timeout: 120000, }); const s3config = { @@ -82,22 +83,24 @@ const s3 = { putBucketEncryption: params => wrap(() => s3Client.send(new PutBucketEncryptionCommand(params))), getBucketEncryption: params => wrap(() => s3Client.send(new GetBucketEncryptionCommand(params))), putObject: params => wrap(() => s3Client.send(new PutObjectCommand(params))), - getObject: params => wrap(async () => { - const response = await s3Client.send(new GetObjectCommand(params)); - const body = await response.Body.transformToString(); - return { ...response, Body: body }; - }), + getObject: params => + wrap(async () => { + const response = await s3Client.send(new GetObjectCommand(params)); + const body = await response.Body.transformToString(); + return { ...response, Body: body }; + }), listBuckets: params => wrap(() => s3Client.send(new ListBucketsCommand(params || {}))), copyObject: params => wrap(() => s3Client.send(new CopyObjectCommand(params))), - listObjectVersions: params => wrap(async () => { - const response = await s3Client.send(new ListObjectVersionsCommand(params)); - return { - ...response, - Versions: response.Versions || [], - DeleteMarkers: response.DeleteMarkers || [], - CommonPrefixes: response.CommonPrefixes || [] - }; - }), + listObjectVersions: params => + wrap(async () => { + const response = await s3Client.send(new ListObjectVersionsCommand(params)); + return { + ...response, + Versions: response.Versions || [], + DeleteMarkers: response.DeleteMarkers || [], + CommonPrefixes: response.CommonPrefixes || [], + }; + }), headObject: params => wrap(() => s3Client.send(new HeadObjectCommand(params))), createMultipartUpload: params => wrap(() => s3Client.send(new CreateMultipartUploadCommand(params))), uploadPart: params => wrap(() => s3Client.send(new UploadPartCommand(params))), @@ -105,14 +108,15 @@ const s3 = { completeMultipartUpload: params => wrap(() => s3Client.send(new CompleteMultipartUploadCommand(params))), putBucketVersioning: params => wrap(() => s3Client.send(new PutBucketVersioningCommand(params))), headBucket: params => wrap(() => s3Client.send(new HeadBucketCommand(params))), - listMultipartUploads: params => wrap(async () => { - const response = await s3Client.send(new ListMultipartUploadsCommand(params)); - return { - ...response, - Uploads: response.Uploads || [], - CommonPrefixes: response.CommonPrefixes || [] - }; - }), + listMultipartUploads: params => + wrap(async () => { + const response = await s3Client.send(new ListMultipartUploadsCommand(params)); + return { + ...response, + Uploads: response.Uploads || [], + CommonPrefixes: response.CommonPrefixes || [], + }; + }), listParts: params => wrap(() => s3Client.send(new ListPartsCommand(params))), _compat: bucketUtil.s3, config: { @@ -129,13 +133,18 @@ const s3 = { function hydrateSSEConfig({ algo: SSEAlgorithm, masterKeyId: KMSMasterKeyID }) { // Stringify and parse to strip undefined values - return JSON.parse(JSON.stringify({ Rules: [{ - ApplyServerSideEncryptionByDefault: { - SSEAlgorithm, - KMSMasterKeyID, - }, - }], - })); + return JSON.parse( + JSON.stringify({ + Rules: [ + { + ApplyServerSideEncryptionByDefault: { + SSEAlgorithm, + KMSMasterKeyID, + }, + }, + ], + }), + ); } function putObjParams(Bucket, Key, sseConfig, kmsKeyId) { diff --git a/tests/functional/sse-kms-migration/load.js b/tests/functional/sse-kms-migration/load.js index 33773fa899..786eb841e7 100644 --- a/tests/functional/sse-kms-migration/load.js +++ b/tests/functional/sse-kms-migration/load.js @@ -37,7 +37,6 @@ async function spawnTcpdump(port, packetCount) { detached: true, stdio: ['ignore', 'pipe', 'pipe'], // ignored stdin shell: false, // no need as it's detached - }, ); let stderr = ''; @@ -51,7 +50,8 @@ async function spawnTcpdump(port, packetCount) { spawnTimeout = setTimeout(() => { if (child.exitCode !== null || child.signalCode !== null) { const err = `countPacketsByIp.sh stopped after spawn with code ${ - child.exitCode} and signal ${child.signalCode}.\nStderr: ${stderr}`; + child.exitCode + } and signal ${child.signalCode}.\nStderr: ${stderr}`; reject(new Error(err)); } else { resolve(child); @@ -70,9 +70,7 @@ async function spawnTcpdump(port, packetCount) { if (spawnTimeout) { clearTimeout(spawnTimeout); } - reject(new Error( - `tcpdump script closed with code ${code} and signal ${signal}.\nStderr: ${stderr}` - )); + reject(new Error(`tcpdump script closed with code ${code} and signal ${signal}.\nStderr: ${stderr}`)); } }); }); @@ -89,8 +87,9 @@ async function stopTcpdump(tcpdump) { }); } -describe(`KMS load (kmip cluster ${KMS_NODES} nodes): ${OBJECT_NUMBER - } objs each in ${BUCKET_NUMBER} bkts (${TOTAL_OBJECTS} objs)`, () => { +describe(`KMS load (kmip cluster ${KMS_NODES} nodes): ${ + OBJECT_NUMBER +} objs each in ${BUCKET_NUMBER} bkts (${TOTAL_OBJECTS} objs)`, () => { let buckets = []; let tcpdumpProcess; let stdout; @@ -107,18 +106,23 @@ describe(`KMS load (kmip cluster ${KMS_NODES} nodes): ${OBJECT_NUMBER await helpers.s3.putBucketEncryption({ Bucket, ServerSideEncryptionConfiguration: helpers.hydrateSSEConfig({ - algo: 'aws:kms', masterKeyId: masterKeyArn }), + algo: 'aws:kms', + masterKeyId: masterKeyArn, + }), }); return { Bucket, masterKeyArn }; - })); + }), + ); }); after(async () => { - await Promise.all(buckets.map(async ({ Bucket, masterKeyArn }) => { - await helpers.cleanup(Bucket); - return helpers.destroyKmsKey(masterKeyArn, log); - })); + await Promise.all( + buckets.map(async ({ Bucket, masterKeyArn }) => { + await helpers.cleanup(Bucket); + return helpers.destroyKmsKey(masterKeyArn, log); + }), + ); await promisify(kms.client.stop.bind(kms.client))(); }); @@ -147,7 +151,7 @@ describe(`KMS load (kmip cluster ${KMS_NODES} nodes): ${OBJECT_NUMBER const [count, ip] = line.trim().split(' '); return { count: +count, ip }; }), - }) + }), ); }); }); @@ -175,30 +179,37 @@ describe(`KMS load (kmip cluster ${KMS_NODES} nodes): ${OBJECT_NUMBER const repartitionCount = repartition.map(({ count }) => count); assert.strictEqual(code, 0, `tcpdump script closed with code ${code} and signal ${signal}`); assert(repartition.length === KMS_NODES, `Expected ${KMS_NODES} IPs but got ${repartition.length}`); - assert(repartitionCount.every(count => - count >= EXPECTED_MIN && count <= EXPECTED_MAX), + assert( + repartitionCount.every(count => count >= EXPECTED_MIN && count <= EXPECTED_MAX), `Repartition counts should be around ${TOTAL_OBJECTS_PER_NODE} ` + - `(±${APPROX}, min: ${EXPECTED_MIN}, max: ${EXPECTED_MAX}) but got ${repartitionCount}`); + `(±${APPROX}, min: ${EXPECTED_MIN}, max: ${EXPECTED_MAX}) but got ${repartitionCount}`, + ); } it(`should encrypt ${TOTAL_OBJECTS} times in parallel, ~${TOTAL_OBJECTS_PER_NODE} per node`, async () => { - await (Promise.all( - buckets.map(async ({ Bucket }) => Promise.all( - // Send little more request in case a packet is missed. - new Array(OBJECT_NUMBER + 1).fill(0).map(async (_, i) => - helpers.s3.putObject({ Bucket, Key: `obj-${i}`, Body: `body-${i}` })) - )) - )); + await Promise.all( + buckets.map(async ({ Bucket }) => + Promise.all( + // Send little more request in case a packet is missed. + new Array(OBJECT_NUMBER + 1) + .fill(0) + .map(async (_, i) => helpers.s3.putObject({ Bucket, Key: `obj-${i}`, Body: `body-${i}` })), + ), + ), + ); await assertRepartition(closePromise); }); it(`should decrypt ${TOTAL_OBJECTS} times in parallel, ~${TOTAL_OBJECTS_PER_NODE} per node`, async () => { await Promise.all( - buckets.map(async ({ Bucket }) => Promise.all( - // Send little more request in case a packet is missed. - new Array(OBJECT_NUMBER + 1).fill(0).map(async (_, i) => - helpers.s3.getObject({ Bucket, Key: `obj-${i}` })) - )) + buckets.map(async ({ Bucket }) => + Promise.all( + // Send little more request in case a packet is missed. + new Array(OBJECT_NUMBER + 1) + .fill(0) + .map(async (_, i) => helpers.s3.getObject({ Bucket, Key: `obj-${i}` })), + ), + ), ); await assertRepartition(closePromise); }); diff --git a/tests/functional/sse-kms-migration/migration.js b/tests/functional/sse-kms-migration/migration.js index f34748c223..7b85e32e57 100644 --- a/tests/functional/sse-kms-migration/migration.js +++ b/tests/functional/sse-kms-migration/migration.js @@ -25,16 +25,17 @@ async function assertObjectSSE( const sseMD = await helpers.getObjectMDSSE(Bucket, Key); const head = await helpers.s3.headObject({ Bucket, Key, VersionId }); const sseMDMigrated = await helpers.getObjectMDSSE(Bucket, Key); - const expectedKey = `${sseMD.SSEKMSKeyId && isScalityKmsArn(sseMD.SSEKMSKeyId) - ? '' : arnPrefix}${sseMD.SSEKMSKeyId}`; + const expectedKey = `${ + sseMD.SSEKMSKeyId && isScalityKmsArn(sseMD.SSEKMSKeyId) ? '' : arnPrefix + }${sseMD.SSEKMSKeyId}`; if (!put && sseMD.SSEKMSKeyId) { assert.doesNotMatch(sseMD.SSEKMSKeyId, SCAL_KMS_ARN_REG); } // obj precedence over bkt - assert.strictEqual(head.ServerSideEncryption, (objConf.algo || bktConf.algo)); - headers && assert.strictEqual(headers.ServerSideEncryption, (objConf.algo || bktConf.algo)); + assert.strictEqual(head.ServerSideEncryption, objConf.algo || bktConf.algo); + headers && assert.strictEqual(headers.ServerSideEncryption, objConf.algo || bktConf.algo); if (sseMDMigrated.SSEKMSKeyId) { // on metadata verify the full key with arn prefix @@ -82,12 +83,10 @@ describe('SSE KMS migration', () => { bkts[bktConf.name] = bkt; if (bktConf.algo && bktConf.masterKeyId) { bkt.kmsKeyInfo = await helpers.createKmsKey(log); - bkt.kmsKey = bktConf.arnPrefix - ? bkt.kmsKeyInfo.masterKeyArn - : bkt.kmsKeyInfo.masterKeyId; + bkt.kmsKey = bktConf.arnPrefix ? bkt.kmsKeyInfo.masterKeyArn : bkt.kmsKeyInfo.masterKeyId; } - await helpers.s3.headBucket(({ Bucket: bkt.name })); - await helpers.s3.headBucket(({ Bucket: bkt.vname })); + await helpers.s3.headBucket({ Bucket: bkt.name }); + await helpers.s3.headBucket({ Bucket: bkt.vname }); if (bktConf.algo) { const bktSSE = await helpers.getBucketSSE(bkt.name); assert.strictEqual(bktSSE.SSEAlgorithm, bktConf.algo); @@ -103,38 +102,40 @@ describe('SSE KMS migration', () => { } // Check object SSE using MD api, not S3 to avoid triggering migration - await Promise.all(scenarios.testCases.map(async objConf => { - const obj = { - name: `for-copy-enc-obj-${objConf.name}`, - kmsKeyInfo: null, - kmsKey: null, - body: `BODY(for-copy-enc-obj-${objConf.name})`, - }; - bkt.objs[objConf.name] = obj; - if (objConf.algo && objConf.masterKeyId) { - obj.kmsKeyInfo = await helpers.createKmsKey(log); - obj.kmsKey = objConf.arnPrefix - ? obj.kmsKeyInfo.masterKeyArn - : obj.kmsKeyInfo.masterKeyId; - } - const objSSE = await helpers.getObjectMDSSE(bkt.name, obj.name); - assert.strictEqual(objSSE.ServerSideEncryption, objConf.algo || bktConf.algo || ''); - assert.doesNotMatch(objSSE.SSEKMSKeyId, SCAL_KMS_ARN_REG); - return undefined; - })); + await Promise.all( + scenarios.testCases.map(async objConf => { + const obj = { + name: `for-copy-enc-obj-${objConf.name}`, + kmsKeyInfo: null, + kmsKey: null, + body: `BODY(for-copy-enc-obj-${objConf.name})`, + }; + bkt.objs[objConf.name] = obj; + if (objConf.algo && objConf.masterKeyId) { + obj.kmsKeyInfo = await helpers.createKmsKey(log); + obj.kmsKey = objConf.arnPrefix ? obj.kmsKeyInfo.masterKeyArn : obj.kmsKeyInfo.masterKeyId; + } + const objSSE = await helpers.getObjectMDSSE(bkt.name, obj.name); + assert.strictEqual(objSSE.ServerSideEncryption, objConf.algo || bktConf.algo || ''); + assert.doesNotMatch(objSSE.SSEKMSKeyId, SCAL_KMS_ARN_REG); + return undefined; + }), + ); }; before('setup', async () => { - console.log('Run migration', - { profile: helpers.credsProfile, accessKeyId: helpers.s3.config.credentials.accessKeyId }); + console.log('Run migration', { + profile: helpers.credsProfile, + accessKeyId: helpers.s3.config.credentials.accessKeyId, + }); const allBuckets = (await helpers.s3.listBuckets()).Buckets.map(b => b.Name); console.log('List buckets:', allBuckets); await helpers.MD.setup(); await helpers.s3.headBucket({ Bucket: copyBkt }); - await helpers.s3.headBucket(({ Bucket: mpuCopyBkt })); + await helpers.s3.headBucket({ Bucket: mpuCopyBkt }); const copySSE = await helpers.s3.getBucketEncryption({ Bucket: copyBkt }); - const { SSEAlgorithm, KMSMasterKeyID } = copySSE - .ServerSideEncryptionConfiguration.Rules[0].ApplyServerSideEncryptionByDefault; + const { SSEAlgorithm, KMSMasterKeyID } = + copySSE.ServerSideEncryptionConfiguration.Rules[0].ApplyServerSideEncryptionByDefault; assert.strictEqual(SSEAlgorithm, 'aws:kms'); assert.doesNotMatch(KMSMasterKeyID, SCAL_KMS_ARN_REG); @@ -146,241 +147,289 @@ describe('SSE KMS migration', () => { await helpers.cleanup(copyBkt); await helpers.cleanup(mpuCopyBkt); // Clean every bucket - await Promise.all(Object.values(bkts).map(async bkt => { - await helpers.cleanup(bkt.name); - return await helpers.cleanup(bkt.vname); - })); + await Promise.all( + Object.values(bkts).map(async bkt => { + await helpers.cleanup(bkt.name); + return await helpers.cleanup(bkt.vname); + }), + ); }); - scenarios.testCases.forEach(bktConf => describe(`bucket enc-bkt-${bktConf.name}`, () => { - let bkt = bkts[bktConf.name]; - - before(() => { - bkt = bkts[bktConf.name]; - }); - - if (bktConf.deleteSSE) { - beforeEach(async () => scenarios.deleteBucketSSEBeforeEach(bkt.name, log)); - } + scenarios.testCases.forEach(bktConf => + describe(`bucket enc-bkt-${bktConf.name}`, () => { + let bkt = bkts[bktConf.name]; - if (!bktConf.algo) { - it('GetBucketEncryption should return ServerSideEncryptionConfigurationNotFoundError', - async () => await scenarios.tests.getBucketSSEError(bkt.name)); + before(() => { + bkt = bkts[bktConf.name]; + }); - if (!bktConf.deleteSSE) { - it('should have non mandatory SSE in bucket MD as test init put an object with AES256', - async () => scenarios.tests.getBucketNonMandatorySSE(bkt.name, log, 'migration')); + if (bktConf.deleteSSE) { + beforeEach(async () => scenarios.deleteBucketSSEBeforeEach(bkt.name, log)); } - } else { - it('ensure old SSE KMS key setup', - async () => await scenarios.tests.getBucketSSE(bkt.name, log, bktConf.algo, - bktConf.masterKeyId ? bkt.kmsKeyInfo.masterKeyArn : null, 'migration')); - } - scenarios.testCasesObj.forEach(objConf => it(`should have pre uploaded object with SSE ${objConf.name}`, - async () => { - const obj = bkt.objs[objConf.name]; - // use MD here to avoid triggering a migration - const sseMD = await helpers.getObjectMDSSE(bkt.name, obj.name); - if (sseMD.SSEKMSKeyId) { - assert.doesNotMatch(sseMD.SSEKMSKeyId, SCAL_KMS_ARN_REG); - } - })); - - scenarios.testCasesObj.forEach(objConf => describe(`object enc-obj-${objConf.name}`, () => { - const obj = { - name: `enc-obj-${objConf.name}`, - kmsKeyInfo: null, - kmsKey: null, - body: `BODY(enc-obj-${objConf.name})`, - }; - /** to be used as source of copy */ - let objForCopy; - - before(async () => { - if (objConf.algo && objConf.masterKeyId) { - obj.kmsKeyInfo = await helpers.createKmsKey(log); - obj.kmsKey = objConf.arnPrefix - ? obj.kmsKeyInfo.masterKeyArn - : obj.kmsKeyInfo.masterKeyId; - } - objForCopy = bkt.objs[objConf.name]; - }); + if (!bktConf.algo) { + it('GetBucketEncryption should return ServerSideEncryptionConfigurationNotFoundError', async () => + await scenarios.tests.getBucketSSEError(bkt.name)); - const mpus = {}; - before('retrieve MPUS', async () => { - const listed = await helpers.s3.listMultipartUploads({ Bucket: bkt.name }); - assert.strictEqual(listed.IsTruncated, false, 'Too much MPUs, need to loop on pagination'); - for (const mpu of listed.Uploads) { - mpus[mpu.Key] = mpu.UploadId; + if (!bktConf.deleteSSE) { + it('should have non mandatory SSE in bucket MD as test init put an object with AES256', async () => + scenarios.tests.getBucketNonMandatorySSE(bkt.name, log, 'migration')); } - }); - - it(`should PutObject ${obj.name} overriding bucket SSE`, async () => { - await helpers.putEncryptedObject(bkt.name, obj.name, objConf, obj.kmsKey, obj.body); - const assertion = { - Bucket: bkt.name, - Key: obj.name, - Body: obj.body, - }; - await assertObjectSSE(assertion, { objConf, obj }, { bktConf, bkt }, { put: true }); - }); - - // CopyObject scenarios - [ - { name: `${obj.name} into encrypted destination bucket`, forceBktSSE: true }, - { name: `${obj.name} into same bucket with object SSE config` }, - { name: `from encrypted source into ${obj.name} with object SSE config` }, - ].forEach(({ name, forceBktSSE }, index) => - it(`should CopyObject ${name}`, async () => - await scenarios.tests.copyObjectAndSSE( - { copyBkt, objForCopy, copyObj }, - { objConf, obj }, - { bktConf, bkt }, - { index, forceBktSSE, assertObjectSSEFct: assertObjectSSE }, - ))); - - // S3C-9996 The SSE was bugged with MPU, where the completion takes only the masterKeyId from bucket - // Fixed at the same time as migration, some scenario can pass only in newer version above migration - const optionalSkip = objConf.algo || bktConf.masterKeyId || (!bktConf.algo && !bktConf.deleteSSE) - ? it.skip - : it; - - // completed MPU should behave like regular objects - [ - { name: '', keySuffix: '', body: `${obj.body}-MPU1${obj.body}-MPU2` }, - { name: 'that has copy', keySuffix: 'copy', body: `BODY(copy)${obj.body}-MPU2` }, - { name: 'that has byte range copy', keySuffix: 'copyrange', body: 'copyBODY' }, - ].forEach(({ name, keySuffix, body }) => - optionalSkip(`should migrate completed MPU ${name}`, async () => { - const mpuKey = `${obj.name}-mpu${keySuffix}`; - const assertion = { Bucket: bkt.name, Key: mpuKey, Body: body }; - await assertObjectSSE( - assertion, { objConf, obj }, { bktConf, bkt }, fileArnPrefix); - })); - - async function prepareMPUTest(mpuKey, expectedExistingParts) { - const uploadId = mpus[mpuKey]; - assert(uploadId, 'Missing MPU, it should have been prepared before'); - const MPUBucketName = `${mpuBucketPrefix}${bkt.name}`; - const longMPUIdentifier = `overview${splitter}${mpuKey}${splitter}${uploadId}`; - const mpuOverviewMDSSE = await helpers.getObjectMDSSE(MPUBucketName, longMPUIdentifier); - - const existingParts = await helpers.s3.listParts({ - Bucket: bkt.name, Key: mpuKey, UploadId: uploadId }); - const partCount = (existingParts.Parts || []).length || 0; - assert.strictEqual(existingParts.IsTruncated, false, 'Too much parts, need to loop on pagination'); - assert.strictEqual(partCount, expectedExistingParts); - return { mpuKey, uploadId, mpuOverviewMDSSE, partCount, existingParts: existingParts.Parts || [] }; + } else { + it('ensure old SSE KMS key setup', async () => + await scenarios.tests.getBucketSSE( + bkt.name, + log, + bktConf.algo, + bktConf.masterKeyId ? bkt.kmsKeyInfo.masterKeyArn : null, + 'migration', + )); } - // ongoing MPU with regular uploadPart - [ - { - name: 'empty', - keySuffix: '-empty', - existingPartsCount: 0, - partsBody: [`${obj.body}-MPU1`, `${obj.body}-MPU2`], - body: `${obj.body}-MPU1${obj.body}-MPU2`, - }, - { - name: 'with 2 parts', - keySuffix: '', - existingPartsCount: 2, - partsBody: [`${obj.body}-MPU1`, `${obj.body}-MPU2`], - body: `${obj.body}-MPU1${obj.body}-MPU2`.repeat(2), - }, - ].forEach(({ name, keySuffix, existingPartsCount, partsBody, body }) => - optionalSkip(`should finish ongoing encrypted MPU ${name} by adding 2 parts`, async () => { - const { mpuKey, uploadId, mpuOverviewMDSSE, existingParts, partCount } = - await prepareMPUTest(`${obj.name}-migration-mpu${keySuffix}`, existingPartsCount); - const newParts = []; - for (const [index, body] of partsBody.entries()) { - const part = await scenarios.tests.mpuUploadPart({ - UploadId: uploadId, + scenarios.testCasesObj.forEach(objConf => + it(`should have pre uploaded object with SSE ${objConf.name}`, async () => { + const obj = bkt.objs[objConf.name]; + // use MD here to avoid triggering a migration + const sseMD = await helpers.getObjectMDSSE(bkt.name, obj.name); + if (sseMD.SSEKMSKeyId) { + assert.doesNotMatch(sseMD.SSEKMSKeyId, SCAL_KMS_ARN_REG); + } + }), + ); + + scenarios.testCasesObj.forEach(objConf => + describe(`object enc-obj-${objConf.name}`, () => { + const obj = { + name: `enc-obj-${objConf.name}`, + kmsKeyInfo: null, + kmsKey: null, + body: `BODY(enc-obj-${objConf.name})`, + }; + /** to be used as source of copy */ + let objForCopy; + + before(async () => { + if (objConf.algo && objConf.masterKeyId) { + obj.kmsKeyInfo = await helpers.createKmsKey(log); + obj.kmsKey = objConf.arnPrefix ? obj.kmsKeyInfo.masterKeyArn : obj.kmsKeyInfo.masterKeyId; + } + objForCopy = bkt.objs[objConf.name]; + }); + + const mpus = {}; + before('retrieve MPUS', async () => { + const listed = await helpers.s3.listMultipartUploads({ Bucket: bkt.name }); + assert.strictEqual(listed.IsTruncated, false, 'Too much MPUs, need to loop on pagination'); + for (const mpu of listed.Uploads) { + mpus[mpu.Key] = mpu.UploadId; + } + }); + + it(`should PutObject ${obj.name} overriding bucket SSE`, async () => { + await helpers.putEncryptedObject(bkt.name, obj.name, objConf, obj.kmsKey, obj.body); + const assertion = { + Bucket: bkt.name, + Key: obj.name, + Body: obj.body, + }; + await assertObjectSSE(assertion, { objConf, obj }, { bktConf, bkt }, { put: true }); + }); + + // CopyObject scenarios + [ + { name: `${obj.name} into encrypted destination bucket`, forceBktSSE: true }, + { name: `${obj.name} into same bucket with object SSE config` }, + { name: `from encrypted source into ${obj.name} with object SSE config` }, + ].forEach(({ name, forceBktSSE }, index) => + it(`should CopyObject ${name}`, async () => + await scenarios.tests.copyObjectAndSSE( + { copyBkt, objForCopy, copyObj }, + { objConf, obj }, + { bktConf, bkt }, + { index, forceBktSSE, assertObjectSSEFct: assertObjectSSE }, + )), + ); + + // S3C-9996 The SSE was bugged with MPU, where the completion takes only the masterKeyId from bucket + // Fixed at the same time as migration, some scenario can pass only in newer version above migration + const optionalSkip = + objConf.algo || bktConf.masterKeyId || (!bktConf.algo && !bktConf.deleteSSE) ? it.skip : it; + + // completed MPU should behave like regular objects + [ + { name: '', keySuffix: '', body: `${obj.body}-MPU1${obj.body}-MPU2` }, + { name: 'that has copy', keySuffix: 'copy', body: `BODY(copy)${obj.body}-MPU2` }, + { name: 'that has byte range copy', keySuffix: 'copyrange', body: 'copyBODY' }, + ].forEach(({ name, keySuffix, body }) => + optionalSkip(`should migrate completed MPU ${name}`, async () => { + const mpuKey = `${obj.name}-mpu${keySuffix}`; + const assertion = { Bucket: bkt.name, Key: mpuKey, Body: body }; + await assertObjectSSE(assertion, { objConf, obj }, { bktConf, bkt }, fileArnPrefix); + }), + ); + + async function prepareMPUTest(mpuKey, expectedExistingParts) { + const uploadId = mpus[mpuKey]; + assert(uploadId, 'Missing MPU, it should have been prepared before'); + const MPUBucketName = `${mpuBucketPrefix}${bkt.name}`; + const longMPUIdentifier = `overview${splitter}${mpuKey}${splitter}${uploadId}`; + const mpuOverviewMDSSE = await helpers.getObjectMDSSE(MPUBucketName, longMPUIdentifier); + + const existingParts = await helpers.s3.listParts({ Bucket: bkt.name, - Body: body, Key: mpuKey, - PartNumber: partCount + index + 1, - }, mpuOverviewMDSSE, objConf.algo || bktConf.algo); - newParts.push(part); + UploadId: uploadId, + }); + const partCount = (existingParts.Parts || []).length || 0; + assert.strictEqual( + existingParts.IsTruncated, + false, + 'Too much parts, need to loop on pagination', + ); + assert.strictEqual(partCount, expectedExistingParts); + return { + mpuKey, + uploadId, + mpuOverviewMDSSE, + partCount, + existingParts: existingParts.Parts || [], + }; } - await scenarios.tests.mpuComplete( - { UploadId: uploadId, Bucket: bkt.name, Key: mpuKey }, - { existingParts, newParts }, - mpuOverviewMDSSE, objConf.algo || bktConf.algo); - const assertion = { - Bucket: bkt.name, - Key: mpuKey, - Body: body, - }; - await assertObjectSSE( - assertion, { objConf, obj }, { bktConf, bkt }, fileArnPrefix); - })); - - optionalSkip('should finish ongoing encrypted MPU with 2 parts by copy and upload part', async () => { - const { mpuKey, uploadId, mpuOverviewMDSSE, existingParts, partCount } = - await prepareMPUTest(`${obj.name}-migration-mpucopy`, 2); - const part1 = await scenarios.tests.mpuUploadPartCopy({ - UploadId: uploadId, - Bucket: bkt.name, - Key: mpuKey, - PartNumber: partCount + 1, - CopySource: `${copyBkt}/${copyObj}`, - }, mpuOverviewMDSSE, objConf.algo || bktConf.algo); - const part2 = await scenarios.tests.mpuUploadPart({ - UploadId: uploadId, - Bucket: bkt.name, - Body: `${obj.body}-MPU2`, - Key: mpuKey, - PartNumber: partCount + 2, - }, mpuOverviewMDSSE, objConf.algo || bktConf.algo); - await scenarios.tests.mpuComplete( - { UploadId: uploadId, Bucket: bkt.name, Key: mpuKey }, - { existingParts, newParts: [part1, part2] }, - mpuOverviewMDSSE, objConf.algo || bktConf.algo); - const assertion = { - Bucket: bkt.name, - Key: mpuKey, - Body: `BODY(copy)${obj.body}-MPU2`.repeat(2), - }; - await assertObjectSSE( - assertion, { objConf, obj }, { bktConf, bkt }, fileArnPrefix); - }); - - optionalSkip('should finish ongoing encrypted MPU with 2 parts by 2 copy byte range', async () => { - const { mpuKey, uploadId, mpuOverviewMDSSE, existingParts, partCount } = - await prepareMPUTest(`${obj.name}-migration-mpucopyrange`, 2); - // source body is "BODY(copy)" - // [copy, BODY] - const sourceRanges = ['bytes=5-8', 'bytes=0-3']; - const newParts = []; - for (const [index, range] of sourceRanges.entries()) { - const part = await scenarios.tests.mpuUploadPartCopy({ - UploadId: uploadId, - Bucket: bkt.name, - Key: mpuKey, - PartNumber: partCount + index + 1, - CopySource: `${copyBkt}/${copyObj}`, - CopySourceRange: range, - }, mpuOverviewMDSSE, objConf.algo || bktConf.algo); - newParts.push(part); - } - await scenarios.tests.mpuComplete( - { UploadId: uploadId, Bucket: bkt.name, Key: mpuKey }, - { existingParts, newParts }, - mpuOverviewMDSSE, objConf.algo || bktConf.algo); - const assertion = { - Bucket: bkt.name, - Key: mpuKey, - Body: 'copyBODY'.repeat(2), - }; - await assertObjectSSE( - assertion, { objConf, obj }, { bktConf, bkt }, fileArnPrefix); - }); - })); - })); + // ongoing MPU with regular uploadPart + [ + { + name: 'empty', + keySuffix: '-empty', + existingPartsCount: 0, + partsBody: [`${obj.body}-MPU1`, `${obj.body}-MPU2`], + body: `${obj.body}-MPU1${obj.body}-MPU2`, + }, + { + name: 'with 2 parts', + keySuffix: '', + existingPartsCount: 2, + partsBody: [`${obj.body}-MPU1`, `${obj.body}-MPU2`], + body: `${obj.body}-MPU1${obj.body}-MPU2`.repeat(2), + }, + ].forEach(({ name, keySuffix, existingPartsCount, partsBody, body }) => + optionalSkip(`should finish ongoing encrypted MPU ${name} by adding 2 parts`, async () => { + const { mpuKey, uploadId, mpuOverviewMDSSE, existingParts, partCount } = + await prepareMPUTest(`${obj.name}-migration-mpu${keySuffix}`, existingPartsCount); + const newParts = []; + for (const [index, body] of partsBody.entries()) { + const part = await scenarios.tests.mpuUploadPart( + { + UploadId: uploadId, + Bucket: bkt.name, + Body: body, + Key: mpuKey, + PartNumber: partCount + index + 1, + }, + mpuOverviewMDSSE, + objConf.algo || bktConf.algo, + ); + newParts.push(part); + } + await scenarios.tests.mpuComplete( + { UploadId: uploadId, Bucket: bkt.name, Key: mpuKey }, + { existingParts, newParts }, + mpuOverviewMDSSE, + objConf.algo || bktConf.algo, + ); + const assertion = { + Bucket: bkt.name, + Key: mpuKey, + Body: body, + }; + await assertObjectSSE(assertion, { objConf, obj }, { bktConf, bkt }, fileArnPrefix); + }), + ); + + optionalSkip( + 'should finish ongoing encrypted MPU with 2 parts by copy and upload part', + async () => { + const { mpuKey, uploadId, mpuOverviewMDSSE, existingParts, partCount } = + await prepareMPUTest(`${obj.name}-migration-mpucopy`, 2); + const part1 = await scenarios.tests.mpuUploadPartCopy( + { + UploadId: uploadId, + Bucket: bkt.name, + Key: mpuKey, + PartNumber: partCount + 1, + CopySource: `${copyBkt}/${copyObj}`, + }, + mpuOverviewMDSSE, + objConf.algo || bktConf.algo, + ); + const part2 = await scenarios.tests.mpuUploadPart( + { + UploadId: uploadId, + Bucket: bkt.name, + Body: `${obj.body}-MPU2`, + Key: mpuKey, + PartNumber: partCount + 2, + }, + mpuOverviewMDSSE, + objConf.algo || bktConf.algo, + ); + await scenarios.tests.mpuComplete( + { UploadId: uploadId, Bucket: bkt.name, Key: mpuKey }, + { existingParts, newParts: [part1, part2] }, + mpuOverviewMDSSE, + objConf.algo || bktConf.algo, + ); + const assertion = { + Bucket: bkt.name, + Key: mpuKey, + Body: `BODY(copy)${obj.body}-MPU2`.repeat(2), + }; + await assertObjectSSE(assertion, { objConf, obj }, { bktConf, bkt }, fileArnPrefix); + }, + ); + + optionalSkip('should finish ongoing encrypted MPU with 2 parts by 2 copy byte range', async () => { + const { mpuKey, uploadId, mpuOverviewMDSSE, existingParts, partCount } = await prepareMPUTest( + `${obj.name}-migration-mpucopyrange`, + 2, + ); + // source body is "BODY(copy)" + // [copy, BODY] + const sourceRanges = ['bytes=5-8', 'bytes=0-3']; + const newParts = []; + for (const [index, range] of sourceRanges.entries()) { + const part = await scenarios.tests.mpuUploadPartCopy( + { + UploadId: uploadId, + Bucket: bkt.name, + Key: mpuKey, + PartNumber: partCount + index + 1, + CopySource: `${copyBkt}/${copyObj}`, + CopySourceRange: range, + }, + mpuOverviewMDSSE, + objConf.algo || bktConf.algo, + ); + newParts.push(part); + } + + await scenarios.tests.mpuComplete( + { UploadId: uploadId, Bucket: bkt.name, Key: mpuKey }, + { existingParts, newParts }, + mpuOverviewMDSSE, + objConf.algo || bktConf.algo, + ); + const assertion = { + Bucket: bkt.name, + Key: mpuKey, + Body: 'copyBODY'.repeat(2), + }; + await assertObjectSSE(assertion, { objConf, obj }, { bktConf, bkt }, fileArnPrefix); + }); + }), + ); + }), + ); it('should finish ongoing encrypted MPU by copy parts from all bkt and objects matrice', async () => { const mpuKey = 'mpucopy'; @@ -403,18 +452,20 @@ describe('SSE KMS migration', () => { const uploadPromises = scenarios.testCases.reduce((acc, bktConf, bktIdx) => { const bkt = bkts[bktConf.name]; - return acc.concat(scenarios.testCasesObj.map(async (objConf, objIdx) => { - const obj = bkt.objs[objConf.name]; + return acc.concat( + scenarios.testCasesObj.map(async (objConf, objIdx) => { + const obj = bkt.objs[objConf.name]; - const partNumber = partCount + bktIdx * scenarios.testCasesObj.length + objIdx + 1; - const res = await helpers.s3.uploadPartCopy({ - ...copyPartArg, - PartNumber: partNumber, - CopySource: `${bkt.name}/${obj.name}`, - }); + const partNumber = partCount + bktIdx * scenarios.testCasesObj.length + objIdx + 1; + const res = await helpers.s3.uploadPartCopy({ + ...copyPartArg, + PartNumber: partNumber, + CopySource: `${bkt.name}/${obj.name}`, + }); - return { partNumber, body: obj.body, res: res.CopyPartResult }; - })); + return { partNumber, body: obj.body, res: res.CopyPartResult }; + }), + ); }, []); const parts = await Promise.all(uploadPromises); @@ -436,6 +487,10 @@ describe('SSE KMS migration', () => { Body: parts.reduce((acc, part) => `${acc}${part.body}`, '').repeat(2), }; await assertObjectSSE( - assertion, { objConf: {}, obj: {} }, { bktConf: { algo: 'AES256' }, bkt: {} }, fileArnPrefix); + assertion, + { objConf: {}, obj: {} }, + { bktConf: { algo: 'AES256' }, bkt: {} }, + fileArnPrefix, + ); }); }); diff --git a/tests/functional/sse-kms-migration/scenarios.js b/tests/functional/sse-kms-migration/scenarios.js index 008a5c7491..8d344256e2 100644 --- a/tests/functional/sse-kms-migration/scenarios.js +++ b/tests/functional/sse-kms-migration/scenarios.js @@ -52,10 +52,10 @@ async function assertObjectSSE( const sseMD = await helpers.getObjectMDSSE(Bucket, Key); const arnPrefixReg = new RegExp(`^${arnPrefix}`); - const expectedAlgo = (objConf.algo || bktConf.algo) || - (testCase === 'after' && helpers.config.globalEncryptionEnabled && !bktConf.deleteSSE - ? 'AES256' - : undefined); + const expectedAlgo = + objConf.algo || + bktConf.algo || + (testCase === 'after' && helpers.config.globalEncryptionEnabled && !bktConf.deleteSSE ? 'AES256' : undefined); // obj precedence over bkt assert.strictEqual(head.ServerSideEncryption, expectedAlgo); @@ -209,8 +209,7 @@ async function copyObjectAndSSE( const { SSEAlgorithm, KMSMasterKeyID } = await helpers.getBucketSSE(copyBkt); assert.strictEqual(headers.ServerSideEncryption, SSEAlgorithm); testCase !== 'before' && assert.strictEqual(headers.SSEKMSKeyId, KMSMasterKeyID); - const keyArn = `${KMSMasterKeyID && isScalityKmsArn(KMSMasterKeyID) - ? '' : kms.arnPrefix}${KMSMasterKeyID}`; + const keyArn = `${KMSMasterKeyID && isScalityKmsArn(KMSMasterKeyID) ? '' : kms.arnPrefix}${KMSMasterKeyID}`; const kmsKeyInfo = { masterKeyId: getKeyIdFromArn(keyArn), masterKeyArn: keyArn, @@ -263,7 +262,9 @@ async function mpuUploadPart({ UploadId, Bucket, Key, Body, PartNumber }, mpuOve // before has no headers to assert async function mpuUploadPartCopy( { UploadId, Bucket, Key, PartNumber, CopySource, CopySourceRange }, - mpuOverviewMDSSE, algo, testCase + mpuOverviewMDSSE, + algo, + testCase, ) { const part = await helpers.s3.uploadPartCopy({ UploadId, @@ -284,23 +285,23 @@ async function mpuComplete({ UploadId, Bucket, Key }, { existingParts, newParts assert(eTag !== undefined, `Could not find ETag in part: ${JSON.stringify(part)}`); return eTag; }; - + const allParts = [ - ...existingParts.map(part => ({ - PartNumber: part.PartNumber, - ETag: extractETag(part) + ...existingParts.map(part => ({ + PartNumber: part.PartNumber, + ETag: extractETag(part), })), - ...newParts.map((part, idx) => ({ - PartNumber: existingParts.length + idx + 1, - ETag: extractETag(part) + ...newParts.map((part, idx) => ({ + PartNumber: existingParts.length + idx + 1, + ETag: extractETag(part), })), - ]; + ]; const complete = await helpers.s3.completeMultipartUpload({ UploadId, Bucket, Key, MultipartUpload: { - Parts: allParts, + Parts: allParts, }, }); testCase !== 'before' && assertMPUSSEHeaders(complete, mpuOverviewMDSSE, algo); diff --git a/tests/functional/utilities/reportHandler.js b/tests/functional/utilities/reportHandler.js index dc5948eeb7..fe49f8bb91 100644 --- a/tests/functional/utilities/reportHandler.js +++ b/tests/functional/utilities/reportHandler.js @@ -142,16 +142,16 @@ function requestHandler(req, res) { } } else { switch (req.url) { - case '/_/crr/status': - case '/_/ingestion/status': - res.write(JSON.stringify(expectedStatusResults)); - break; - case '/_/crr/resume/all': - case '/_/ingestion/resume/all': - res.write(JSON.stringify(expectedScheduleResults)); - break; - default: - break; + case '/_/crr/status': + case '/_/ingestion/status': + res.write(JSON.stringify(expectedStatusResults)); + break; + case '/_/crr/resume/all': + case '/_/ingestion/resume/all': + res.write(JSON.stringify(expectedScheduleResults)); + break; + default: + break; } } res.end(); @@ -168,8 +168,7 @@ function requestHandler(req, res) { describe('Test Request Failure Cases', () => { before(done => { - httpServer = http.createServer(requestFailHandler) - .listen(testPort); + httpServer = http.createServer(requestFailHandler).listen(testPort); httpServer.on('listening', done); httpServer.on('error', err => { process.stdout.write(`https server: ${err.stack}\n`); @@ -181,8 +180,7 @@ function requestHandler(req, res) { httpServer.close(); }); - it('should return empty object if a request error occurs', - done => { + it('should return empty object if a request error occurs', done => { const endpoint = 'http://nonexists:4242'; item.method(endpoint, 'all', logger, (err, res) => { assert.ifError(err); @@ -191,8 +189,7 @@ function requestHandler(req, res) { }); }); - it('should return empty object if response status code is >= 400', - done => { + it('should return empty object if response status code is >= 400', done => { const endpoint = 'http://localhost:4242'; item.method(endpoint, 'all', logger, (err, res) => { assert.ifError(err); @@ -205,8 +202,7 @@ function requestHandler(req, res) { describe('Test Request Success Cases', () => { const endpoint = 'http://localhost:4242'; before(done => { - httpServer = http.createServer(requestHandler) - .listen(testPort); + httpServer = http.createServer(requestHandler).listen(testPort); httpServer.on('listening', done); httpServer.on('error', err => { process.stdout.write(`https server: ${err.stack}\n`); @@ -221,8 +217,7 @@ function requestHandler(req, res) { it('should return correct location metrics', done => { item.method(endpoint, 'site1', logger, (err, res) => { assert.ifError(err); - assert.deepStrictEqual( - res, item.result.byLocation.site1); + assert.deepStrictEqual(res, item.result.byLocation.site1); done(); }); }); @@ -256,27 +251,33 @@ function requestHandler(req, res) { it('should return correct results', done => { if (item.method.name === 'getIngestionMetrics') { const sites = ['site1', 'site2']; - item.method(sites, logger, (err, res) => { - assert.ifError(err); - assert.deepStrictEqual(res, item.result); - done(); - }, config); + item.method( + sites, + logger, + (err, res) => { + assert.ifError(err); + assert.deepStrictEqual(res, item.result); + done(); + }, + config, + ); } else { - item.method(logger, (err, res) => { - assert.ifError(err); - assert.deepStrictEqual(res, item.result); - done(); - }, config); + item.method( + logger, + (err, res) => { + assert.ifError(err); + assert.deepStrictEqual(res, item.result); + done(); + }, + config, + ); } }); }); }); }); -[ - { method: getReplicationStates }, - { method: getIngestionStates }, -].forEach(item => { +[{ method: getReplicationStates }, { method: getIngestionStates }].forEach(item => { describe(`reportHandler::${item.method.name}`, function testSuite() { this.timeout(20000); const testPort = '4242'; @@ -284,8 +285,7 @@ function requestHandler(req, res) { describe('Test Request Failure Cases', () => { before(done => { - httpServer = http.createServer(requestFailHandler) - .listen(testPort); + httpServer = http.createServer(requestFailHandler).listen(testPort); httpServer.on('listening', done); httpServer.on('error', err => { process.stdout.write(`https server: ${err.stack}\n`); @@ -297,29 +297,34 @@ function requestHandler(req, res) { httpServer.close(); }); - it('should return empty object if a request error occurs', - done => { - item.method(logger, (err, res) => { - assert.ifError(err); - assert.deepStrictEqual(res, {}); - done(); - }, { backbeat: { host: 'nonexisthost', port: testPort } }); + it('should return empty object if a request error occurs', done => { + item.method( + logger, + (err, res) => { + assert.ifError(err); + assert.deepStrictEqual(res, {}); + done(); + }, + { backbeat: { host: 'nonexisthost', port: testPort } }, + ); }); - it('should return empty object if response status code is >= 400', - done => { - item.method(logger, (err, res) => { - assert.ifError(err); - assert.deepStrictEqual(res, {}); - done(); - }, { backbeat: { host: 'localhost', port: testPort } }); + it('should return empty object if response status code is >= 400', done => { + item.method( + logger, + (err, res) => { + assert.ifError(err); + assert.deepStrictEqual(res, {}); + done(); + }, + { backbeat: { host: 'localhost', port: testPort } }, + ); }); }); describe('Test Request Success Cases', () => { before(done => { - httpServer = http.createServer(requestHandler) - .listen(testPort); + httpServer = http.createServer(requestHandler).listen(testPort); httpServer.on('listening', done); httpServer.on('error', err => { process.stdout.write(`https server: ${err.stack}\n`); @@ -332,20 +337,24 @@ function requestHandler(req, res) { }); it('should return correct results', done => { - item.method(logger, (err, res) => { - const expectedResults = { - states: { - site1: 'enabled', - site2: 'disabled', - }, - schedules: { - site2: expectedScheduleResults.site2, - }, - }; - assert.ifError(err); - assert.deepStrictEqual(res, expectedResults); - done(); - }, { backbeat: { host: 'localhost', port: testPort } }); + item.method( + logger, + (err, res) => { + const expectedResults = { + states: { + site1: 'enabled', + site2: 'disabled', + }, + schedules: { + site2: expectedScheduleResults.site2, + }, + }; + assert.ifError(err); + assert.deepStrictEqual(res, expectedResults); + done(); + }, + { backbeat: { host: 'localhost', port: testPort } }, + ); }); }); }); @@ -358,8 +367,7 @@ describe('reportHanlder::getIngestionInfo', function testSuite() { describe('Test Request Success Cases', () => { before(done => { - httpServer = http.createServer(requestHandler) - .listen(testPort); + httpServer = http.createServer(requestHandler).listen(testPort); httpServer.on('listening', done); httpServer.on('error', err => { process.stdout.write(`https server: ${err.stack}\n`); @@ -372,25 +380,28 @@ describe('reportHanlder::getIngestionInfo', function testSuite() { }); it('should return correct results', done => { - getIngestionInfo(logger, (err, res) => { - const expectedStatusResults = { - states: { - site1: 'enabled', - site2: 'disabled', - }, - schedules: { - site2: expectedScheduleResults.site2, - }, - }; - assert.ifError(err); + getIngestionInfo( + logger, + (err, res) => { + const expectedStatusResults = { + states: { + site1: 'enabled', + site2: 'disabled', + }, + schedules: { + site2: expectedScheduleResults.site2, + }, + }; + assert.ifError(err); - assert(res.metrics); - assert(res.status); - assert.deepStrictEqual(res.status, expectedStatusResults); - assert.deepStrictEqual(res.metrics, - ingestionExpectedResultsRef); - done(); - }, config); + assert(res.metrics); + assert(res.status); + assert.deepStrictEqual(res.status, expectedStatusResults); + assert.deepStrictEqual(res.metrics, ingestionExpectedResultsRef); + done(); + }, + config, + ); }); it('should return empty if no ingestion locations exist', done => { diff --git a/tests/locationConfig/locationConfigLegacy.json b/tests/locationConfig/locationConfigLegacy.json index 73dfed5179..c078a81ace 100644 --- a/tests/locationConfig/locationConfigLegacy.json +++ b/tests/locationConfig/locationConfigLegacy.json @@ -1,4 +1,3 @@ - { "legacy": { "type": "mem", diff --git a/tests/multipleBackend/backendHealthcheckResponse.js b/tests/multipleBackend/backendHealthcheckResponse.js index d24c4e3d70..b5b85fe253 100644 --- a/tests/multipleBackend/backendHealthcheckResponse.js +++ b/tests/multipleBackend/backendHealthcheckResponse.js @@ -2,8 +2,7 @@ const assert = require('assert'); const DummyRequestLogger = require('../unit/helpers').DummyRequestLogger; -const clientCheck - = require('../../lib/utilities/healthcheckHandler').clientCheck; +const clientCheck = require('../../lib/utilities/healthcheckHandler').clientCheck; const { config } = require('../../lib/Config'); const { getAzureClient, @@ -11,10 +10,7 @@ const { getAzureContainerName, } = require('../functional/aws-node-sdk/test/multipleBackend/utils'); -const { - LOCATION_NAME_DMF, - LOCATION_NAME_CRR, -} = require('../constants'); +const { LOCATION_NAME_DMF, LOCATION_NAME_CRR } = require('../constants'); const log = new DummyRequestLogger(); const locConstraints = Object.keys(config.locationConstraints); @@ -23,55 +19,59 @@ const azureClient = getAzureClient(); describe('Healthcheck response', function describeHealthcheck() { this.timeout(60000); - it('should return result for every location constraint in ' + - 'locationConfig and every external locations with flightCheckOnStartUp ' + - 'set to true', done => { - clientCheck(true, log, (err, results) => { - const resultKeys = Object.keys(results); - locConstraints.forEach(constraint => { - if (constraint === LOCATION_NAME_DMF || constraint === LOCATION_NAME_CRR) { - // FIXME: location-dmf-v1 and location-crr-v1 are not in results, see CLDSRV-440 - return; - } - assert(resultKeys.includes(constraint), `constraint: ${constraint} not in results: ${resultKeys}`); + it( + 'should return result for every location constraint in ' + + 'locationConfig and every external locations with flightCheckOnStartUp ' + + 'set to true', + done => { + clientCheck(true, log, (err, results) => { + const resultKeys = Object.keys(results); + locConstraints.forEach(constraint => { + if (constraint === LOCATION_NAME_DMF || constraint === LOCATION_NAME_CRR) { + // FIXME: location-dmf-v1 and location-crr-v1 are not in results, see CLDSRV-440 + return; + } + assert(resultKeys.includes(constraint), `constraint: ${constraint} not in results: ${resultKeys}`); + }); + done(); }); - done(); - }); - }); - it('should return no error with flightCheckOnStartUp set to false', - done => { + }, + ); + it('should return no error with flightCheckOnStartUp set to false', done => { clientCheck(false, log, err => { - assert.strictEqual(err, null, - `Expected success but got error ${err}`); + assert.strictEqual(err, null, `Expected success but got error ${err}`); done(); }); }); - it('should return result for every location constraint in ' + - 'locationConfig and at least one of every external locations with ' + - 'flightCheckOnStartUp set to false', done => { - clientCheck(false, log, (err, results) => { - assert.notStrictEqual(results.length, locConstraints.length); - locConstraints.forEach(constraint => { - if (constraint === LOCATION_NAME_DMF || constraint === LOCATION_NAME_CRR) { - // FIXME: location-dmf-v1 and location-crr-v1 are not in results, see CLDSRV-440 - return; - } - if (Object.keys(results).indexOf(constraint) === -1) { - const locationType = config - .locationConstraints[constraint].type; - assert(Object.keys(results).some(result => - config.locationConstraints[result].type - === locationType)); - } + it( + 'should return result for every location constraint in ' + + 'locationConfig and at least one of every external locations with ' + + 'flightCheckOnStartUp set to false', + done => { + clientCheck(false, log, (err, results) => { + assert.notStrictEqual(results.length, locConstraints.length); + locConstraints.forEach(constraint => { + if (constraint === LOCATION_NAME_DMF || constraint === LOCATION_NAME_CRR) { + // FIXME: location-dmf-v1 and location-crr-v1 are not in results, see CLDSRV-440 + return; + } + if (Object.keys(results).indexOf(constraint) === -1) { + const locationType = config.locationConstraints[constraint].type; + assert( + Object.keys(results).some( + result => config.locationConstraints[result].type === locationType, + ), + ); + } + }); + done(); }); - done(); - }); - }); + }, + ); // FIXME: does not pass, see CLDSRV-441 describe.skip('Azure container creation', () => { - const containerName = - getAzureContainerName(azureLocationNonExistContainer); + const containerName = getAzureContainerName(azureLocationNonExistContainer); beforeEach(async () => { await azureClient.getContainerClient(containerName).deleteIfExists(); @@ -81,44 +81,57 @@ describe('Healthcheck response', function describeHealthcheck() { await azureClient.getContainerClient(containerName).deleteIfExists(); }); - it('should create an azure location\'s container if it is missing ' + - 'and the check is a flightCheckOnStartUp', done => { - clientCheck(true, log, (err, results) => { - const azureLocationNonExistContainerError = - results[azureLocationNonExistContainer].error; - if (err) { - assert(err.is.InternalError, `got unexpected err in clientCheck: ${err}`); - assert(azureLocationNonExistContainerError.startsWith( - 'The specified container is being deleted.')); - return done(); - } - return azureClient.getContainerClient(containerName).getProperties( - azureResult => { - assert.strictEqual(azureResult.metadata.name, containerName); - return done(); - }, err => { - assert.strictEqual(err, null, 'got unexpected err ' + - `heading azure container: ${err}`); + it( + "should create an azure location's container if it is missing " + 'and the check is a flightCheckOnStartUp', + done => { + clientCheck(true, log, (err, results) => { + const azureLocationNonExistContainerError = results[azureLocationNonExistContainer].error; + if (err) { + assert(err.is.InternalError, `got unexpected err in clientCheck: ${err}`); + assert( + azureLocationNonExistContainerError.startsWith('The specified container is being deleted.'), + ); return done(); - }); - }); - }); + } + return azureClient.getContainerClient(containerName).getProperties( + azureResult => { + assert.strictEqual(azureResult.metadata.name, containerName); + return done(); + }, + err => { + assert.strictEqual(err, null, 'got unexpected err ' + `heading azure container: ${err}`); + return done(); + }, + ); + }); + }, + ); - it('should not create an azure location\'s container even if it is ' + - 'missing if the check is not a flightCheckOnStartUp', done => { - clientCheck(false, log, err => { - assert.strictEqual(err, null, - `got unexpected err in clientCheck: ${err}`); - return azureClient.getContainerClient(containerName).getProperties().then( - () => { - assert(err, 'Expected err but did not find one'); - return done(); - }, err => { - assert.strictEqual(err.code, 'NotFound', - `got unexpected err code in clientCheck: ${err.code}`); - return done(); - }); - }); - }); + it( + "should not create an azure location's container even if it is " + + 'missing if the check is not a flightCheckOnStartUp', + done => { + clientCheck(false, log, err => { + assert.strictEqual(err, null, `got unexpected err in clientCheck: ${err}`); + return azureClient + .getContainerClient(containerName) + .getProperties() + .then( + () => { + assert(err, 'Expected err but did not find one'); + return done(); + }, + err => { + assert.strictEqual( + err.code, + 'NotFound', + `got unexpected err code in clientCheck: ${err.code}`, + ); + return done(); + }, + ); + }); + }, + ); }); }); diff --git a/tests/multipleBackend/multipartUpload.js b/tests/multipleBackend/multipartUpload.js index b5d537d73d..8edf6b6f8f 100644 --- a/tests/multipleBackend/multipartUpload.js +++ b/tests/multipleBackend/multipartUpload.js @@ -1,17 +1,12 @@ const assert = require('assert'); const async = require('async'); -const { S3Client, - HeadObjectCommand, - AbortMultipartUploadCommand, - ListPartsCommand } = require('@aws-sdk/client-s3'); +const { S3Client, HeadObjectCommand, AbortMultipartUploadCommand, ListPartsCommand } = require('@aws-sdk/client-s3'); const { parseString } = require('xml2js'); const { models } = require('arsenal'); const BucketInfo = models.BucketInfo; -const { getRealAwsConfig } = - require('../functional/aws-node-sdk/test/support/awsConfig'); -const { cleanup, DummyRequestLogger, makeAuthInfo, versioningTestUtils } = - require('../unit/helpers'); +const { getRealAwsConfig } = require('../functional/aws-node-sdk/test/support/awsConfig'); +const { cleanup, DummyRequestLogger, makeAuthInfo, versioningTestUtils } = require('../unit/helpers'); const DummyRequest = require('../unit/DummyRequest'); const { config } = require('../../lib/Config'); const { metadata } = require('arsenal').storage.metadata.inMemory.metadata; @@ -20,13 +15,11 @@ const { bucketPut } = require('../../lib/api/bucketPut'); const objectPut = require('../../lib/api/objectPut'); const objectGet = require('../../lib/api/objectGet'); const bucketPutVersioning = require('../../lib/api/bucketPutVersioning'); -const initiateMultipartUpload = - require('../../lib/api/initiateMultipartUpload'); +const initiateMultipartUpload = require('../../lib/api/initiateMultipartUpload'); const multipartDelete = require('../../lib/api/multipartDelete'); const objectPutCopyPart = require('../../lib/api/objectPutCopyPart'); const objectPutPart = require('../../lib/api/objectPutPart'); -const completeMultipartUpload = - require('../../lib/api/completeMultipartUpload'); +const completeMultipartUpload = require('../../lib/api/completeMultipartUpload'); const listParts = require('../../lib/api/listParts'); const listMultipartUploads = require('../../lib/api/listMultipartUploads'); const constants = require('../../constants'); @@ -47,8 +40,7 @@ const namespace = 'default'; const bucketName = 'bucketname'; const mpuBucket = `${constants.mpuBucketPrefix}${bucketName}`; const awsBucket = config.locationConstraints[awsLocation].details.bucketName; -const awsMismatchBucket = config.locationConstraints[awsLocationMismatch] - .details.bucketName; +const awsMismatchBucket = config.locationConstraints[awsLocationMismatch].details.bucketName; const smallBody = Buffer.from('I am a body', 'utf8'); const bigBody = Buffer.alloc(10485760); const locMetaHeader = 'scal-location-constraint'; @@ -65,7 +57,8 @@ const bucketPutRequest = { const awsETag = 'be747eb4b75517bf6b3cf7c5fbb62f3a'; const awsETagBigObj = 'f1c9645dbc14efddc7d8a322685f26eb'; const tagSet = 'key1=value1&key2=value2'; -const completeBody = '' + +const completeBody = + '' + '' + '1' + `"${awsETagBigObj}"` + @@ -83,29 +76,38 @@ const basicParams = { }; function getObjectGetRequest(objectKey) { - return Object.assign({ - objectKey, - headers: {}, - url: `/${bucketName}/${objectKey}`, - }, basicParams); + return Object.assign( + { + objectKey, + headers: {}, + url: `/${bucketName}/${objectKey}`, + }, + basicParams, + ); } function getDeleteParams(objectKey, uploadId) { - return Object.assign({ - url: `/${objectKey}?uploadId=${uploadId}`, - query: { uploadId }, - objectKey, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - }, basicParams); + return Object.assign( + { + url: `/${objectKey}?uploadId=${uploadId}`, + query: { uploadId }, + objectKey, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + }, + basicParams, + ); } function getPartParams(objectKey, uploadId, partNumber) { - return Object.assign({ - objectKey, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - url: `/${objectKey}?partNumber=${partNumber}&uploadId=${uploadId}`, - query: { partNumber, uploadId }, - }, basicParams); + return Object.assign( + { + objectKey, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: `/${objectKey}?partNumber=${partNumber}&uploadId=${uploadId}`, + query: { partNumber, uploadId }, + }, + basicParams, + ); } function _getOverviewKey(objectKey, uploadId) { @@ -113,23 +115,29 @@ function _getOverviewKey(objectKey, uploadId) { } function getCompleteParams(objectKey, uploadId) { - return Object.assign({ - objectKey, - parsedHost: 's3.amazonaws.com', - headers: { host: `${bucketName}.s3.amazonaws.com` }, - post: completeBody, - url: `/${objectKey}?uploadId=${uploadId}`, - query: { uploadId }, - }, basicParams); + return Object.assign( + { + objectKey, + parsedHost: 's3.amazonaws.com', + headers: { host: `${bucketName}.s3.amazonaws.com` }, + post: completeBody, + url: `/${objectKey}?uploadId=${uploadId}`, + query: { uploadId }, + }, + basicParams, + ); } function getListParams(objectKey, uploadId) { - return Object.assign({ - objectKey, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - url: `/${objectKey}?uploadId=${uploadId}`, - query: { uploadId }, - }, basicParams); + return Object.assign( + { + objectKey, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: `/${objectKey}?uploadId=${uploadId}`, + query: { uploadId }, + }, + basicParams, + ); } function getAwsParams(objectKey) { @@ -143,10 +151,8 @@ function getAwsParamsBucketNotMatch(objectKey) { function assertMpuInitResults(initResult, key, cb) { parseString(initResult, (err, json) => { assert.equal(err, null, `Error parsing mpu init results: ${err}`); - assert.strictEqual(json.InitiateMultipartUploadResult - .Bucket[0], bucketName); - assert.strictEqual(json.InitiateMultipartUploadResult - .Key[0], key); + assert.strictEqual(json.InitiateMultipartUploadResult.Bucket[0], bucketName); + assert.strictEqual(json.InitiateMultipartUploadResult.Key[0], key); assert(json.InitiateMultipartUploadResult.UploadId[0]); cb(json.InitiateMultipartUploadResult.UploadId[0]); }); @@ -154,16 +160,13 @@ function assertMpuInitResults(initResult, key, cb) { function assertMpuCompleteResults(compResult, objectKey) { parseString(compResult, (err, json) => { - assert.equal(err, null, - `Error parsing mpu complete results: ${err}`); + assert.equal(err, null, `Error parsing mpu complete results: ${err}`); assert.strictEqual( json.CompleteMultipartUploadResult.Location[0], - `http://${bucketName}.s3.amazonaws.com/${objectKey}`); - assert.strictEqual( - json.CompleteMultipartUploadResult.Bucket[0], - bucketName); - assert.strictEqual( - json.CompleteMultipartUploadResult.Key[0], objectKey); + `http://${bucketName}.s3.amazonaws.com/${objectKey}`, + ); + assert.strictEqual(json.CompleteMultipartUploadResult.Bucket[0], bucketName); + assert.strictEqual(json.CompleteMultipartUploadResult.Key[0], objectKey); const MD = metadata.keyMaps.get(bucketName).get(objectKey); assert(MD); }); @@ -174,53 +177,42 @@ function assertListResults(listResult, testAttribute, uploadId, objectKey) { assert.equal(err, null, `Error parsing list part results: ${err}`); assert.strictEqual(json.ListPartsResult.Key[0], objectKey); assert.strictEqual(json.ListPartsResult.UploadId[0], uploadId); - assert.strictEqual(json.ListPartsResult.Initiator[0].ID[0], - authInfo.getCanonicalID()); + assert.strictEqual(json.ListPartsResult.Initiator[0].ID[0], authInfo.getCanonicalID()); // attributes to test specific to PartNumberMarker being set // in listParts if (testAttribute === 'partNumMarker') { - assert.strictEqual(json.ListPartsResult.NextPartNumberMarker, - undefined); + assert.strictEqual(json.ListPartsResult.NextPartNumberMarker, undefined); assert.strictEqual(json.ListPartsResult.IsTruncated[0], 'false'); assert.strictEqual(json.ListPartsResult.Part.length, 1); assert.strictEqual(json.ListPartsResult.PartNumberMarker[0], '1'); // data of second part put assert.strictEqual(json.ListPartsResult.Part[0].PartNumber[0], '2'); - assert.strictEqual(json.ListPartsResult.Part[0].ETag[0], - `"${awsETag}"`); + assert.strictEqual(json.ListPartsResult.Part[0].ETag[0], `"${awsETag}"`); assert.strictEqual(json.ListPartsResult.Part[0].Size[0], '11'); } else { // common attributes to test if MaxParts set or // neither MaxParts nor PartNumberMarker set - assert.strictEqual(json.ListPartsResult.PartNumberMarker, - undefined); + assert.strictEqual(json.ListPartsResult.PartNumberMarker, undefined); assert.strictEqual(json.ListPartsResult.Part[0].PartNumber[0], '1'); - assert.strictEqual(json.ListPartsResult.Part[0].ETag[0], - `"${awsETagBigObj}"`); - assert.strictEqual(json.ListPartsResult.Part[0].Size[0], - '10485760'); + assert.strictEqual(json.ListPartsResult.Part[0].ETag[0], `"${awsETagBigObj}"`); + assert.strictEqual(json.ListPartsResult.Part[0].Size[0], '10485760'); // attributes to test specific to MaxParts being set in listParts if (testAttribute === 'maxParts') { - assert.strictEqual(json.ListPartsResult.NextPartNumberMarker[0], - '1'); + assert.strictEqual(json.ListPartsResult.NextPartNumberMarker[0], '1'); assert.strictEqual(json.ListPartsResult.IsTruncated[0], 'true'); assert.strictEqual(json.ListPartsResult.Part.length, 1); assert.strictEqual(json.ListPartsResult.MaxParts[0], '1'); } else { // attributes to test if neither MaxParts nor // PartNumberMarker set - assert.strictEqual(json.ListPartsResult.NextPartNumberMarker, - undefined); - assert.strictEqual(json.ListPartsResult.IsTruncated[0], - 'false'); + assert.strictEqual(json.ListPartsResult.NextPartNumberMarker, undefined); + assert.strictEqual(json.ListPartsResult.IsTruncated[0], 'false'); assert.strictEqual(json.ListPartsResult.Part.length, 2); assert.strictEqual(json.ListPartsResult.MaxParts[0], '1000'); - assert.strictEqual(json.ListPartsResult.Part[1].PartNumber[0], - '2'); - assert.strictEqual(json.ListPartsResult.Part[1].ETag[0], - `"${awsETag}"`); + assert.strictEqual(json.ListPartsResult.Part[1].PartNumber[0], '2'); + assert.strictEqual(json.ListPartsResult.Part[1].ETag[0], `"${awsETag}"`); assert.strictEqual(json.ListPartsResult.Part[1].Size[0], '11'); } } @@ -237,20 +229,20 @@ function _getZenkoObjectKey(objectKey) { function assertObjOnBackend(expectedBackend, objectKey, cb) { const zenkoObjectKey = _getZenkoObjectKey(objectKey); - return objectGet(authInfo, getObjectGetRequest(zenkoObjectKey), false, log, - async (err, result, metaHeaders) => { + return objectGet(authInfo, getObjectGetRequest(zenkoObjectKey), false, log, async (err, result, metaHeaders) => { assert.equal(err, null, `Error getting object on S3: ${err}`); assert.strictEqual(metaHeaders[`x-amz-meta-${locMetaHeader}`], expectedBackend); if (expectedBackend === awsLocation) { - return s3.send(new HeadObjectCommand({ Bucket: awsBucket, Key: objectKey })) - .then(result => { - assert.strictEqual(result.Metadata[locMetaHeader], awsLocation); - return cb(); - }).catch(err => { - assert.equal(err, null, 'Error on headObject call to AWS: ' + - `${err}`); - return cb(); - }); + return s3 + .send(new HeadObjectCommand({ Bucket: awsBucket, Key: objectKey })) + .then(result => { + assert.strictEqual(result.Metadata[locMetaHeader], awsLocation); + return cb(); + }) + .catch(err => { + assert.equal(err, null, 'Error on headObject call to AWS: ' + `${err}`); + return cb(); + }); } return process.nextTick(cb); }); @@ -275,14 +267,12 @@ function mpuSetup(location, key, cb) { bucketName, namespace, objectKey: key, - headers: { 'host': `${bucketName}.s3.amazonaws.com`, - 'x-amz-meta-scal-location-constraint': location }, + headers: { host: `${bucketName}.s3.amazonaws.com`, 'x-amz-meta-scal-location-constraint': location }, url: `/${key}?uploads`, parsedHost: 'localhost', actionImplicitDenies: false, }; - initiateMultipartUpload(authInfo, initiateRequest, log, - (err, result) => { + initiateMultipartUpload(authInfo, initiateRequest, log, (err, result) => { assert.strictEqual(err, null, 'Error initiating MPU'); assertMpuInitResults(result, key, uploadId => { putParts(uploadId, key, () => { @@ -293,14 +283,17 @@ function mpuSetup(location, key, cb) { } function putObject(putBackend, objectKey, cb) { - const putParams = Object.assign({ - headers: { - 'host': `${bucketName}.s3.amazonaws.com`, - 'x-amz-meta-scal-location-constraint': putBackend, + const putParams = Object.assign( + { + headers: { + host: `${bucketName}.s3.amazonaws.com`, + 'x-amz-meta-scal-location-constraint': putBackend, + }, + url: '/', + objectKey, }, - url: '/', - objectKey, - }, basicParams); + basicParams, + ); const objectPutRequest = new DummyRequest(putParams, smallBody); return objectPut(authInfo, objectPutRequest, undefined, log, err => { assert.equal(err, null, `Error putting object to ${putBackend} ${err}`); @@ -313,22 +306,27 @@ function abortMPU(uploadId, awsParams, cb) { s3.send(new AbortMultipartUploadCommand(abortParams)) .then(() => { cb(); - }).catch(err => { - assert.equal(err, null, `Error aborting MPU: ${err}`); + }) + .catch(err => { + assert.equal(err, null, `Error aborting MPU: ${err}`); cb(); }); } function abortMultipleMpus(backendsInfo, callback) { - async.forEach(backendsInfo, (backend, cb) => { - const delParams = getDeleteParams(backend.key, backend.uploadId); - multipartDelete(authInfo, delParams, log, err => { - cb(err); - }); - }, err => { - assert.equal(err, null, `Error aborting MPU: ${err}`); - callback(); - }); + async.forEach( + backendsInfo, + (backend, cb) => { + const delParams = getDeleteParams(backend.key, backend.uploadId); + multipartDelete(authInfo, delParams, log, err => { + cb(err); + }); + }, + err => { + assert.equal(err, null, `Error aborting MPU: ${err}`); + callback(); + }, + ); } describe('Multipart Upload API with AWS Backend', function mpuTestSuite() { @@ -351,15 +349,16 @@ describe('Multipart Upload API with AWS Backend', function mpuTestSuite() { bucketName, namespace, objectKey, - headers: { 'host': `${bucketName}.s3.amazonaws.com`, - 'x-amz-meta-scal-location-constraint': `${awsLocation}` }, + headers: { + host: `${bucketName}.s3.amazonaws.com`, + 'x-amz-meta-scal-location-constraint': `${awsLocation}`, + }, url: `/${objectKey}?uploads`, parsedHost: 'localhost', actionImplicitDenies: false, }; - initiateMultipartUpload(authInfo, initiateRequest, log, - (err, result) => { + initiateMultipartUpload(authInfo, initiateRequest, log, (err, result) => { assert.strictEqual(err, null, 'Error initiating MPU'); assertMpuInitResults(result, objectKey, uploadId => { abortMPU(uploadId, getAwsParams(objectKey), done); @@ -367,23 +366,22 @@ describe('Multipart Upload API with AWS Backend', function mpuTestSuite() { }); }); - it('should initiate a multipart upload on AWS location with ' + - 'bucketMatch equals false', done => { + it('should initiate a multipart upload on AWS location with ' + 'bucketMatch equals false', done => { const objectKey = `key-${Date.now()}`; const initiateRequest = { bucketName, namespace, objectKey, - headers: { 'host': `${bucketName}.s3.amazonaws.com`, - 'x-amz-meta-scal-location-constraint': - `${awsLocationMismatch}` }, + headers: { + host: `${bucketName}.s3.amazonaws.com`, + 'x-amz-meta-scal-location-constraint': `${awsLocationMismatch}`, + }, url: `/${objectKey}?uploads`, parsedHost: 'localhost', actionImplicitDenies: false, }; - initiateMultipartUpload(authInfo, initiateRequest, log, - (err, result) => { + initiateMultipartUpload(authInfo, initiateRequest, log, (err, result) => { assert.strictEqual(err, null, 'Error initiating MPU'); assertMpuInitResults(result, objectKey, uploadId => { abortMPU(uploadId, getAwsParamsBucketNotMatch(objectKey), done); @@ -398,7 +396,7 @@ describe('Multipart Upload API with AWS Backend', function mpuTestSuite() { namespace, objectKey, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-meta-scal-location-constraint': `${awsLocation}`, 'x-amz-tagging': tagSet, }, @@ -407,8 +405,7 @@ describe('Multipart Upload API with AWS Backend', function mpuTestSuite() { actionImplicitDenies: false, }; - initiateMultipartUpload(authInfo, initiateRequest, log, - (err, result) => { + initiateMultipartUpload(authInfo, initiateRequest, log, (err, result) => { assert.ifError(err); assertMpuInitResults(result, objectKey, uploadId => { abortMPU(uploadId, getAwsParams(objectKey), done); @@ -428,8 +425,7 @@ describe('Multipart Upload API with AWS Backend', function mpuTestSuite() { }); }); - it('should list the parts of a multipart upload on real AWS location ' + - 'with bucketMatch set to false', done => { + it('should list the parts of a multipart upload on real AWS location ' + 'with bucketMatch set to false', done => { const objectKey = `key-${Date.now()}`; mpuSetup(awsLocationMismatch, objectKey, uploadId => { const listParams = getListParams(objectKey, uploadId); @@ -441,8 +437,7 @@ describe('Multipart Upload API with AWS Backend', function mpuTestSuite() { }); }); - it('should only return number of parts equal to specified maxParts', - function itF(done) { + it('should only return number of parts equal to specified maxParts', function itF(done) { this.timeout(90000); const objectKey = `key-${Date.now()}`; mpuSetup(awsLocation, objectKey, uploadId => { @@ -476,7 +471,8 @@ describe('Multipart Upload API with AWS Backend', function mpuTestSuite() { abortMPU(uploadId, getAwsParams(objectKey), () => { const listParams = getListParams(objectKey, uploadId); listParts(authInfo, listParams, log, err => { - const wantedDesc = 'Error returned from AWS: ' + + const wantedDesc = + 'Error returned from AWS: ' + 'The specified upload does not exist. The upload ID ' + 'may be invalid, or the upload may have been aborted' + ' or completed.'; @@ -494,37 +490,44 @@ describe('Multipart Upload API with AWS Backend', function mpuTestSuite() { const delParams = getDeleteParams(objectKey, uploadId); multipartDelete(authInfo, delParams, log, async err => { assert.equal(err, null, `Error aborting MPU: ${err}`); - s3.send(new ListPartsCommand({ - Bucket: awsBucket, - Key: objectKey, - UploadId: uploadId, - })).then(() => { - assert.fail('Expected an error listing parts of aborted MPU'); - }).catch(err => { - assert.strictEqual(err.name, 'NoSuchUpload'); - done(); - }); + s3.send( + new ListPartsCommand({ + Bucket: awsBucket, + Key: objectKey, + UploadId: uploadId, + }), + ) + .then(() => { + assert.fail('Expected an error listing parts of aborted MPU'); + }) + .catch(err => { + assert.strictEqual(err.name, 'NoSuchUpload'); + done(); + }); }); }); }); - it('should abort a multipart upload on real AWS location with' + - 'bucketMatch set to false', done => { + it('should abort a multipart upload on real AWS location with' + 'bucketMatch set to false', done => { const objectKey = `key-${Date.now()}`; mpuSetup(awsLocationMismatch, objectKey, uploadId => { const delParams = getDeleteParams(objectKey, uploadId); multipartDelete(authInfo, delParams, log, async err => { assert.equal(err, null, `Error aborting MPU: ${err}`); - s3.send(new ListPartsCommand({ - Bucket: awsBucket, - Key: `${bucketName}/${objectKey}`, - UploadId: uploadId, - })).then(() => { - assert.fail('Expected an error listing parts of aborted MPU'); - }).catch(err => { - assert.strictEqual(err.name, 'NoSuchUpload'); - done(); - }); + s3.send( + new ListPartsCommand({ + Bucket: awsBucket, + Key: `${bucketName}/${objectKey}`, + UploadId: uploadId, + }), + ) + .then(() => { + assert.fail('Expected an error listing parts of aborted MPU'); + }) + .catch(err => { + assert.strictEqual(err.name, 'NoSuchUpload'); + done(); + }); }); }); }); @@ -539,8 +542,7 @@ describe('Multipart Upload API with AWS Backend', function mpuTestSuite() { }); }); - it('should return ServiceUnavailable if MPU deleted directly from AWS ' + - 'and try to complete from S3', done => { + it('should return ServiceUnavailable if MPU deleted directly from AWS ' + 'and try to complete from S3', done => { const objectKey = `key-${Date.now()}`; mpuSetup(awsLocation, objectKey, uploadId => { abortMPU(uploadId, getAwsParams(objectKey), () => { @@ -557,8 +559,7 @@ describe('Multipart Upload API with AWS Backend', function mpuTestSuite() { const objectKey = `key-${Date.now()}`; mpuSetup(awsLocation, objectKey, uploadId => { const compParams = getCompleteParams(objectKey, uploadId); - completeMultipartUpload(authInfo, compParams, log, - (err, result) => { + completeMultipartUpload(authInfo, compParams, log, (err, result) => { assert.equal(err, null, `Error completing mpu on AWS: ${err}`); assertMpuCompleteResults(result, objectKey); assertObjOnBackend(awsLocation, objectKey, done); @@ -566,31 +567,25 @@ describe('Multipart Upload API with AWS Backend', function mpuTestSuite() { }); }); - it('should complete a multipart upload on real AWS location with ' + - 'bucketMatch set to false', done => { + it('should complete a multipart upload on real AWS location with ' + 'bucketMatch set to false', done => { const objectKey = `key-${Date.now()}`; mpuSetup(awsLocationMismatch, objectKey, uploadId => { const compParams = getCompleteParams(objectKey, uploadId); - completeMultipartUpload(authInfo, compParams, log, - (err, result) => { + completeMultipartUpload(authInfo, compParams, log, (err, result) => { assert.equal(err, null, `Error completing mpu on AWS: ${err}`); assertMpuCompleteResults(result, objectKey); - assertObjOnBackend(awsLocationMismatch, - `${bucketName}/${objectKey}`, done); + assertObjOnBackend(awsLocationMismatch, `${bucketName}/${objectKey}`, done); }); }); }); - it('should complete MPU on AWS with same key as object put to file', - done => { + it('should complete MPU on AWS with same key as object put to file', done => { const objectKey = `key-${Date.now()}`; return putObject(fileLocation, objectKey, () => { mpuSetup(awsLocation, objectKey, uploadId => { const compParams = getCompleteParams(objectKey, uploadId); - completeMultipartUpload(authInfo, compParams, log, - (err, result) => { - assert.equal(err, null, 'Error completing mpu on AWS ' + - `${err}`); + completeMultipartUpload(authInfo, compParams, log, (err, result) => { + assert.equal(err, null, 'Error completing mpu on AWS ' + `${err}`); assertMpuCompleteResults(result, objectKey); assertObjOnBackend(awsLocation, objectKey, done); }); @@ -598,16 +593,13 @@ describe('Multipart Upload API with AWS Backend', function mpuTestSuite() { }); }); - it('should complete MPU on file with same key as object put to AWS', - done => { + it('should complete MPU on file with same key as object put to AWS', done => { const objectKey = `key-${Date.now()}`; putObject(awsLocation, objectKey, () => { mpuSetup(fileLocation, objectKey, uploadId => { const compParams = getCompleteParams(objectKey, uploadId); - completeMultipartUpload(authInfo, compParams, log, - (err, result) => { - assert.equal(err, null, 'Error completing mpu on file ' + - `${err}`); + completeMultipartUpload(authInfo, compParams, log, (err, result) => { + assert.equal(err, null, 'Error completing mpu on file ' + `${err}`); assertMpuCompleteResults(result, objectKey); assertObjOnBackend(fileLocation, objectKey, done); }); @@ -615,28 +607,26 @@ describe('Multipart Upload API with AWS Backend', function mpuTestSuite() { }); }); - it('should be successful initiating MPU on AWS with Scality ' + - 'S3 versioning enabled', done => { + it('should be successful initiating MPU on AWS with Scality ' + 'S3 versioning enabled', done => { const objectKey = `key-${Date.now()}`; // putting null version: put obj before versioning configured putObject(awsLocation, objectKey, () => { - const enableVersioningRequest = versioningTestUtils. - createBucketPutVersioningReq(bucketName, 'Enabled'); + const enableVersioningRequest = versioningTestUtils.createBucketPutVersioningReq(bucketName, 'Enabled'); bucketPutVersioning(authInfo, enableVersioningRequest, log, err => { - assert.equal(err, null, 'Error enabling bucket versioning: ' + - `${err}`); + assert.equal(err, null, 'Error enabling bucket versioning: ' + `${err}`); const initiateRequest = { bucketName, namespace, objectKey, - headers: { 'host': `${bucketName}.s3.amazonaws.com`, - 'x-amz-meta-scal-location-constraint': awsLocation }, + headers: { + host: `${bucketName}.s3.amazonaws.com`, + 'x-amz-meta-scal-location-constraint': awsLocation, + }, url: `/${objectKey}?uploads`, parsedHost: 'localhost', actionImplicitDenies: false, }; - initiateMultipartUpload(authInfo, initiateRequest, log, - err => { + initiateMultipartUpload(authInfo, initiateRequest, log, err => { assert.strictEqual(err, null); done(); }); @@ -647,7 +637,8 @@ describe('Multipart Upload API with AWS Backend', function mpuTestSuite() { it('should return invalidPart error', done => { const objectKey = `key-${Date.now()}`; mpuSetup(awsLocation, objectKey, uploadId => { - const errorBody = '' + + const errorBody = + '' + '' + '1' + `"${awsETag}"` + @@ -668,7 +659,8 @@ describe('Multipart Upload API with AWS Backend', function mpuTestSuite() { it('should return invalidPartOrder error', done => { const objectKey = `key-${Date.now()}`; mpuSetup(awsLocation, objectKey, uploadId => { - const errorBody = '' + + const errorBody = + '' + '' + '2' + `"${awsETag}"` + @@ -693,7 +685,8 @@ describe('Multipart Upload API with AWS Backend', function mpuTestSuite() { const partRequest3 = new DummyRequest(putPartParam, smallBody); objectPutPart(authInfo, partRequest3, undefined, log, err => { assert.equal(err, null, `Error putting part: ${err}`); - const errorBody = '' + + const errorBody = + '' + '' + '1' + `"${awsETagBigObj}"` + @@ -721,57 +714,54 @@ describe('Multipart Upload API with AWS Backend', function mpuTestSuite() { const objectKey = `testkey-${Date.now()}`; const fileKey = `fileKey-${Date.now()}`; const memKey = `memKey-${Date.now()}`; - async.series([ - cb => mpuSetup(fileLocation, fileKey, - fileUploadId => cb(null, fileUploadId)), - cb => mpuSetup(memLocation, memKey, memUploadId => - cb(null, memUploadId)), - cb => mpuSetup(awsLocation, objectKey, awsUploadId => - cb(null, awsUploadId)), - ], (err, uploadIds) => { - assert.equal(err, null, `Error setting up MPUs: ${err}`); - const listMpuParams = { - bucketName, - namespace, - headers: { host: '/' }, - url: `/${bucketName}?uploads`, - query: {}, - actionImplicitDenies: false, - }; - listMultipartUploads(authInfo, listMpuParams, log, - (err, mpuListXml) => { - assert.equal(err, null, `Error listing MPUs: ${err}`); - parseString(mpuListXml, (err, json) => { - const mpuListing = json.ListMultipartUploadsResult.Upload; - assert.strictEqual(fileKey, mpuListing[0].Key[0]); - assert.strictEqual(uploadIds[0], mpuListing[0].UploadId[0]); - assert.strictEqual(memKey, mpuListing[1].Key[0]); - assert.strictEqual(uploadIds[1], mpuListing[1].UploadId[0]); - assert.strictEqual(objectKey, mpuListing[2].Key[0]); - assert.strictEqual(uploadIds[2], mpuListing[2].UploadId[0]); - const backendsInfo = [ - { backend: fileLocation, key: fileKey, - uploadId: uploadIds[0] }, - { backend: memLocation, key: memKey, - uploadId: uploadIds[1] }, - { backend: 'aws', key: objectKey, - uploadId: uploadIds[2] }, - ]; - abortMultipleMpus(backendsInfo, done); + async.series( + [ + cb => mpuSetup(fileLocation, fileKey, fileUploadId => cb(null, fileUploadId)), + cb => mpuSetup(memLocation, memKey, memUploadId => cb(null, memUploadId)), + cb => mpuSetup(awsLocation, objectKey, awsUploadId => cb(null, awsUploadId)), + ], + (err, uploadIds) => { + assert.equal(err, null, `Error setting up MPUs: ${err}`); + const listMpuParams = { + bucketName, + namespace, + headers: { host: '/' }, + url: `/${bucketName}?uploads`, + query: {}, + actionImplicitDenies: false, + }; + listMultipartUploads(authInfo, listMpuParams, log, (err, mpuListXml) => { + assert.equal(err, null, `Error listing MPUs: ${err}`); + parseString(mpuListXml, (err, json) => { + const mpuListing = json.ListMultipartUploadsResult.Upload; + assert.strictEqual(fileKey, mpuListing[0].Key[0]); + assert.strictEqual(uploadIds[0], mpuListing[0].UploadId[0]); + assert.strictEqual(memKey, mpuListing[1].Key[0]); + assert.strictEqual(uploadIds[1], mpuListing[1].UploadId[0]); + assert.strictEqual(objectKey, mpuListing[2].Key[0]); + assert.strictEqual(uploadIds[2], mpuListing[2].UploadId[0]); + const backendsInfo = [ + { backend: fileLocation, key: fileKey, uploadId: uploadIds[0] }, + { backend: memLocation, key: memKey, uploadId: uploadIds[1] }, + { backend: 'aws', key: objectKey, uploadId: uploadIds[2] }, + ]; + abortMultipleMpus(backendsInfo, done); + }); }); - }); - }); + }, + ); }); describe('with mpu initiated on legacy version', () => { beforeEach(function beFn() { this.currentTest.lcObj = config.locationConstraints; - const legacyObj = Object.assign(config.locationConstraints, - { legacy: { + const legacyObj = Object.assign(config.locationConstraints, { + legacy: { type: 'mem', legacyAwsBehavior: true, details: {}, - } }); + }, + }); config.setLocationConstraints(legacyObj); }); @@ -783,40 +773,37 @@ describe('Multipart Upload API with AWS Backend', function mpuTestSuite() { const objectKey = `testkey-${Date.now()}`; mpuSetup('scality-internal-mem', objectKey, uploadId => { const mpuOverviewKey = _getOverviewKey(objectKey, uploadId); - async.waterfall([ - next => { - const bucketMD = BucketInfo.fromObj( - metadata.buckets.get(bucketName)); - const objMD = - metadata.keyMaps.get(mpuBucket).get(mpuOverviewKey); - // remove location constraints to mimic legacy behavior - bucketMD.setLocationConstraint(undefined); - objMD.controllingLocationConstraint = undefined; - objMD.dataStoreName = undefined; - objMD[constants.objectLocationConstraintHeader] = - undefined; - next(null, uploadId, bucketMD, objMD); - }, - (uploadId, bucketMD, objMD, next) => { - metadata.buckets.set(bucketName, bucketMD); - metadata.keyMaps.get(mpuBucket). - set(mpuOverviewKey, objMD); - next(null, uploadId); - }, - (uploadId, next) => { - const compParams = getCompleteParams( - objectKey, uploadId); - completeMultipartUpload( - authInfo, compParams, log, next); - }, - (completeRes, resHeaders, next) => { - assertMpuCompleteResults(completeRes, objectKey); - next(); + async.waterfall( + [ + next => { + const bucketMD = BucketInfo.fromObj(metadata.buckets.get(bucketName)); + const objMD = metadata.keyMaps.get(mpuBucket).get(mpuOverviewKey); + // remove location constraints to mimic legacy behavior + bucketMD.setLocationConstraint(undefined); + objMD.controllingLocationConstraint = undefined; + objMD.dataStoreName = undefined; + objMD[constants.objectLocationConstraintHeader] = undefined; + next(null, uploadId, bucketMD, objMD); + }, + (uploadId, bucketMD, objMD, next) => { + metadata.buckets.set(bucketName, bucketMD); + metadata.keyMaps.get(mpuBucket).set(mpuOverviewKey, objMD); + next(null, uploadId); + }, + (uploadId, next) => { + const compParams = getCompleteParams(objectKey, uploadId); + completeMultipartUpload(authInfo, compParams, log, next); + }, + (completeRes, resHeaders, next) => { + assertMpuCompleteResults(completeRes, objectKey); + next(); + }, + ], + err => { + assert.ifError(err); + done(); }, - ], err => { - assert.ifError(err); - done(); - }); + ); }); }); @@ -824,39 +811,37 @@ describe('Multipart Upload API with AWS Backend', function mpuTestSuite() { const objectKey = `testkey-${Date.now()}`; mpuSetup('scality-internal-mem', objectKey, uploadId => { const mpuOverviewKey = _getOverviewKey(objectKey, uploadId); - async.waterfall([ - next => { - const bucketMD = BucketInfo.fromObj( - metadata.buckets.get(bucketName)); - const objMD = - metadata.keyMaps.get(mpuBucket).get(mpuOverviewKey); - // remove location constraints to mimic legacy behavior - bucketMD.setLocationConstraint(undefined); - objMD.controllingLocationConstraint = undefined; - objMD.dataStoreName = undefined; - objMD[constants.objectLocationConstraintHeader] = - undefined; - metadata.buckets.set(bucketName, bucketMD); - metadata.keyMaps.get(mpuBucket). - set(mpuOverviewKey, objMD); - next(null, uploadId); - }, - (uploadId, next) => { - const delParams = getDeleteParams(objectKey, uploadId); - multipartDelete(authInfo, delParams, log, - err => next(err, uploadId)); - }, - (uploadId, next) => { - const listParams = getListParams(objectKey, uploadId); - listParts(authInfo, listParams, log, err => { - assert(err.is.NoSuchUpload); - next(); - }); + async.waterfall( + [ + next => { + const bucketMD = BucketInfo.fromObj(metadata.buckets.get(bucketName)); + const objMD = metadata.keyMaps.get(mpuBucket).get(mpuOverviewKey); + // remove location constraints to mimic legacy behavior + bucketMD.setLocationConstraint(undefined); + objMD.controllingLocationConstraint = undefined; + objMD.dataStoreName = undefined; + objMD[constants.objectLocationConstraintHeader] = undefined; + metadata.buckets.set(bucketName, bucketMD); + metadata.keyMaps.get(mpuBucket).set(mpuOverviewKey, objMD); + next(null, uploadId); + }, + (uploadId, next) => { + const delParams = getDeleteParams(objectKey, uploadId); + multipartDelete(authInfo, delParams, log, err => next(err, uploadId)); + }, + (uploadId, next) => { + const listParams = getListParams(objectKey, uploadId); + listParts(authInfo, listParams, log, err => { + assert(err.is.NoSuchUpload); + next(); + }); + }, + ], + err => { + assert.ifError(err); + done(); }, - ], err => { - assert.ifError(err); - done(); - }); + ); }); }); @@ -864,40 +849,38 @@ describe('Multipart Upload API with AWS Backend', function mpuTestSuite() { const objectKey = `testkey-${Date.now()}`; mpuSetup('scality-internal-mem', objectKey, uploadId => { const mpuOverviewKey = _getOverviewKey(objectKey, uploadId); - async.waterfall([ - next => { - const bucketMD = BucketInfo.fromObj( - metadata.buckets.get(bucketName)); - const objMD = - metadata.keyMaps.get(mpuBucket).get(mpuOverviewKey); - // remove location constraints to mimic legacy behavior - bucketMD.setLocationConstraint(undefined); - objMD.controllingLocationConstraint = undefined; - objMD.dataStoreName = undefined; - objMD[constants.objectLocationConstraintHeader] = - undefined; - metadata.buckets.set(bucketName, bucketMD); - metadata.keyMaps.get(mpuBucket). - set(mpuOverviewKey, objMD); - next(null, uploadId); - }, - (uploadId, next) => { - const listParams = getListParams(objectKey, uploadId); - listParts(authInfo, listParams, log, (err, res) => { - assert.ifError(err); - assertListResults(res, null, uploadId, objectKey); + async.waterfall( + [ + next => { + const bucketMD = BucketInfo.fromObj(metadata.buckets.get(bucketName)); + const objMD = metadata.keyMaps.get(mpuBucket).get(mpuOverviewKey); + // remove location constraints to mimic legacy behavior + bucketMD.setLocationConstraint(undefined); + objMD.controllingLocationConstraint = undefined; + objMD.dataStoreName = undefined; + objMD[constants.objectLocationConstraintHeader] = undefined; + metadata.buckets.set(bucketName, bucketMD); + metadata.keyMaps.get(mpuBucket).set(mpuOverviewKey, objMD); next(null, uploadId); - }); - }, - (uploadId, next) => { - const delParams = getDeleteParams(objectKey, uploadId); - multipartDelete(authInfo, delParams, log, - err => next(err, uploadId)); + }, + (uploadId, next) => { + const listParams = getListParams(objectKey, uploadId); + listParts(authInfo, listParams, log, (err, res) => { + assert.ifError(err); + assertListResults(res, null, uploadId, objectKey); + next(null, uploadId); + }); + }, + (uploadId, next) => { + const delParams = getDeleteParams(objectKey, uploadId); + multipartDelete(authInfo, delParams, log, err => next(err, uploadId)); + }, + ], + err => { + assert.ifError(err); + done(); }, - ], err => { - assert.ifError(err); - done(); - }); + ); }); }); @@ -905,70 +888,58 @@ describe('Multipart Upload API with AWS Backend', function mpuTestSuite() { const objectKey = `testkey-${Date.now()}`; mpuSetup('scality-internal-mem', objectKey, uploadId => { const mpuOverviewKey = _getOverviewKey(objectKey, uploadId); - async.waterfall([ - next => { - const copyObjectKey = `copykey-${Date.now()}`; - putObject('scality-internal-mem', copyObjectKey, - () => next(null, copyObjectKey)); - }, - (copyObjectKey, next) => { - const bucketMD = BucketInfo.fromObj( - metadata.buckets.get(bucketName)); - const mpuMD = - metadata.keyMaps.get(mpuBucket).get(mpuOverviewKey); - const copyObjMD = - metadata.keyMaps.get(bucketName).get(copyObjectKey); - // remove location constraints to mimic legacy behavior - bucketMD.setLocationConstraint(undefined); - mpuMD.controllingLocationConstraint = undefined; - mpuMD.dataStoreName = undefined; - mpuMD[constants.objectLocationConstraintHeader] = - undefined; - copyObjMD.controllingLocationConstraint = undefined; - copyObjMD.dataStoreName = undefined; - copyObjMD[constants.objectLocationConstraintHeader] = - undefined; - metadata.buckets.set(bucketName, bucketMD); - metadata.keyMaps.get(mpuBucket). - set(mpuOverviewKey, mpuMD); - metadata.keyMaps.get(bucketName). - set(copyObjectKey, copyObjMD); - next(null, uploadId, copyObjectKey); - }, - (uploadId, copyObjectKey, next) => { - const copyParams = - getPartParams(objectKey, uploadId, 3); - objectPutCopyPart(authInfo, copyParams, bucketName, - copyObjectKey, undefined, log, err => { - next(err, uploadId); - }); - }, - (uploadId, next) => { - const listParams = getListParams(objectKey, uploadId); - listParts(authInfo, listParams, log, (err, listRes) => { - assert.ifError(err); - parseString(listRes, (err, json) => { - assert.equal(err, null, - `Error parsing list part results: ${err}`); - assert.strictEqual(json.ListPartsResult. - Part[2].PartNumber[0], '3'); - assert.strictEqual(json.ListPartsResult. - Part[2].ETag[0], `"${awsETag}"`); - assert.strictEqual(json.ListPartsResult. - Part[2].Size[0], '11'); - next(null, uploadId); + async.waterfall( + [ + next => { + const copyObjectKey = `copykey-${Date.now()}`; + putObject('scality-internal-mem', copyObjectKey, () => next(null, copyObjectKey)); + }, + (copyObjectKey, next) => { + const bucketMD = BucketInfo.fromObj(metadata.buckets.get(bucketName)); + const mpuMD = metadata.keyMaps.get(mpuBucket).get(mpuOverviewKey); + const copyObjMD = metadata.keyMaps.get(bucketName).get(copyObjectKey); + // remove location constraints to mimic legacy behavior + bucketMD.setLocationConstraint(undefined); + mpuMD.controllingLocationConstraint = undefined; + mpuMD.dataStoreName = undefined; + mpuMD[constants.objectLocationConstraintHeader] = undefined; + copyObjMD.controllingLocationConstraint = undefined; + copyObjMD.dataStoreName = undefined; + copyObjMD[constants.objectLocationConstraintHeader] = undefined; + metadata.buckets.set(bucketName, bucketMD); + metadata.keyMaps.get(mpuBucket).set(mpuOverviewKey, mpuMD); + metadata.keyMaps.get(bucketName).set(copyObjectKey, copyObjMD); + next(null, uploadId, copyObjectKey); + }, + (uploadId, copyObjectKey, next) => { + const copyParams = getPartParams(objectKey, uploadId, 3); + objectPutCopyPart(authInfo, copyParams, bucketName, copyObjectKey, undefined, log, err => { + next(err, uploadId); }); - }); - }, - (uploadId, next) => { - const delParams = getDeleteParams(objectKey, uploadId); - multipartDelete(authInfo, delParams, log, - err => next(err, uploadId)); + }, + (uploadId, next) => { + const listParams = getListParams(objectKey, uploadId); + listParts(authInfo, listParams, log, (err, listRes) => { + assert.ifError(err); + parseString(listRes, (err, json) => { + assert.equal(err, null, `Error parsing list part results: ${err}`); + assert.strictEqual(json.ListPartsResult.Part[2].PartNumber[0], '3'); + assert.strictEqual(json.ListPartsResult.Part[2].ETag[0], `"${awsETag}"`); + assert.strictEqual(json.ListPartsResult.Part[2].Size[0], '11'); + next(null, uploadId); + }); + }); + }, + (uploadId, next) => { + const delParams = getDeleteParams(objectKey, uploadId); + multipartDelete(authInfo, delParams, log, err => next(err, uploadId)); + }, + ], + err => { + assert.ifError(err); + done(); }, - ], err => { - assert.ifError(err); - done(); - }); + ); }); }); }); diff --git a/tests/multipleBackend/objectCopy.js b/tests/multipleBackend/objectCopy.js index 8e59a1617f..66e147c597 100644 --- a/tests/multipleBackend/objectCopy.js +++ b/tests/multipleBackend/objectCopy.js @@ -6,8 +6,7 @@ const objectPut = require('../../lib/api/objectPut'); const objectCopy = require('../../lib/api/objectCopy'); const { metadata } = require('arsenal').storage.metadata.inMemory.metadata; const DummyRequest = require('../unit/DummyRequest'); -const { cleanup, DummyRequestLogger, makeAuthInfo } - = require('../unit/helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../unit/helpers'); const log = new DummyRequestLogger(); const canonicalID = 'accessKey1'; @@ -19,11 +18,13 @@ const memLocation = 'scality-internal-mem'; const fileLocation = 'scality-internal-file'; function _createBucketPutRequest(bucketName, bucketLoc) { - const post = bucketLoc ? '' + - '' + - `${bucketLoc}` + - '' : ''; + const post = bucketLoc + ? '' + + '' + + `${bucketLoc}` + + '' + : ''; return new DummyRequest({ bucketName, namespace, @@ -56,20 +57,18 @@ function _createObjectPutRequest(bucketName, objectKey, body) { } function copySetup(params, cb) { - const { sourceBucket, sourceLocation, sourceKey, destBucket, - destLocation, body } = params; - const putDestBucketRequest = - _createBucketPutRequest(destBucket, destLocation); - const putSourceBucketRequest = - _createBucketPutRequest(sourceBucket, sourceLocation); - const putSourceObjRequest = _createObjectPutRequest(sourceBucket, - sourceKey, body); - async.series([ - callback => bucketPut(authInfo, putDestBucketRequest, log, callback), - callback => bucketPut(authInfo, putSourceBucketRequest, log, callback), - callback => objectPut(authInfo, putSourceObjRequest, undefined, log, - callback), - ], err => cb(err)); + const { sourceBucket, sourceLocation, sourceKey, destBucket, destLocation, body } = params; + const putDestBucketRequest = _createBucketPutRequest(destBucket, destLocation); + const putSourceBucketRequest = _createBucketPutRequest(sourceBucket, sourceLocation); + const putSourceObjRequest = _createObjectPutRequest(sourceBucket, sourceKey, body); + async.series( + [ + callback => bucketPut(authInfo, putDestBucketRequest, log, callback), + callback => bucketPut(authInfo, putSourceBucketRequest, log, callback), + callback => objectPut(authInfo, putSourceObjRequest, undefined, log, callback), + ], + err => cb(err), + ); } describe('ObjectCopy API with multiple backends', () => { @@ -79,29 +78,30 @@ describe('ObjectCopy API with multiple backends', () => { after(() => cleanup()); - it('object metadata for newly stored object should have dataStoreName ' + - 'if copying to mem based on bucket location', done => { - const params = { - sourceBucket: sourceBucketName, - sourceKey: `sourcekey-${Date.now()}`, - sourceLocation: fileLocation, - body: 'testbody', - destBucket: destBucketName, - destLocation: memLocation, - }; - const destKey = `destkey-${Date.now()}`; - const testObjectCopyRequest = - _createObjectCopyRequest(destBucketName, destKey); - copySetup(params, err => { - assert.strictEqual(err, null, `Error setting up copy: ${err}`); - objectCopy(authInfo, testObjectCopyRequest, sourceBucketName, - params.sourceKey, undefined, log, err => { + it( + 'object metadata for newly stored object should have dataStoreName ' + + 'if copying to mem based on bucket location', + done => { + const params = { + sourceBucket: sourceBucketName, + sourceKey: `sourcekey-${Date.now()}`, + sourceLocation: fileLocation, + body: 'testbody', + destBucket: destBucketName, + destLocation: memLocation, + }; + const destKey = `destkey-${Date.now()}`; + const testObjectCopyRequest = _createObjectCopyRequest(destBucketName, destKey); + copySetup(params, err => { + assert.strictEqual(err, null, `Error setting up copy: ${err}`); + objectCopy(authInfo, testObjectCopyRequest, sourceBucketName, params.sourceKey, undefined, log, err => { assert.strictEqual(err, null, `Error copying: ${err}`); const bucket = metadata.keyMaps.get(params.destBucket); const objMd = bucket.get(destKey); assert.strictEqual(objMd.dataStoreName, memLocation); done(); }); - }); - }); + }); + }, + ); }); diff --git a/tests/multipleBackend/objectPut.js b/tests/multipleBackend/objectPut.js index e76dc24cdc..df31ee0a83 100644 --- a/tests/multipleBackend/objectPut.js +++ b/tests/multipleBackend/objectPut.js @@ -2,8 +2,7 @@ const assert = require('assert'); const async = require('async'); const { storage } = require('arsenal'); -const { cleanup, DummyRequestLogger, makeAuthInfo } - = require('../unit/helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../unit/helpers'); const { bucketPut } = require('../../lib/api/bucketPut'); const objectPut = require('../../lib/api/objectPut'); const DummyRequest = require('../unit/DummyRequest'); @@ -24,11 +23,13 @@ const sproxydLocation = 'scality-internal-sproxyd'; const describeSkipIfE2E = process.env.S3_END_TO_END ? describe.skip : describe; function put(bucketLoc, objLoc, requestHost, objectName, cb, errorDescription) { - const post = bucketLoc ? '' + - '' + - `${bucketLoc}` + - '' : ''; + const post = bucketLoc + ? '' + + '' + + `${bucketLoc}` + + '' + : ''; const bucketPutReq = new DummyRequest({ bucketName, namespace, @@ -57,8 +58,7 @@ function put(bucketLoc, objLoc, requestHost, objectName, cb, errorDescription) { testPutObjReq.parsedHost = requestHost; } bucketPut(authInfo, bucketPutReq, log, () => { - objectPut(authInfo, testPutObjReq, undefined, log, (err, - resHeaders) => { + objectPut(authInfo, testPutObjReq, undefined, log, (err, resHeaders) => { if (errorDescription) { assert.strictEqual(err.code, 400); assert(err.is.InvalidArgument); @@ -130,8 +130,7 @@ describeSkipIfE2E('objectPutAPI with multiple backends', function testSuite() { }); function isDataStoredInMem(testCase) { - return testCase.objLoc === memLocation - || (testCase.objLoc === null && testCase.bucketLoc === memLocation); + return testCase.objLoc === memLocation || (testCase.objLoc === null && testCase.bucketLoc === memLocation); } function checkPut(testCase) { @@ -148,28 +147,31 @@ describeSkipIfE2E('objectPutAPI with multiple backends', function testSuite() { putCases.forEach(testCase => { it(`should put an object to ${testCase.name}`, done => { - async.series([ - next => put(testCase.bucketLoc, testCase.objLoc, 'localhost', 'obj1', next), - next => { - checkPut(testCase); - // Increase the probability of the first request having released - // the socket, so that it can be reused for the next request. - // This tests how HTTP connection reuse behaves. - setTimeout(next, 10); - }, - // Second put should work as well - next => put(testCase.bucketLoc, testCase.objLoc, 'localhost', 'obj2', next), - next => { - checkPut(testCase); - setTimeout(next, 10); - }, - // Overwriting PUT - next => put(testCase.bucketLoc, testCase.objLoc, 'localhost', 'obj2', next), - next => { - checkPut(testCase); - next(); - }, - ], done); + async.series( + [ + next => put(testCase.bucketLoc, testCase.objLoc, 'localhost', 'obj1', next), + next => { + checkPut(testCase); + // Increase the probability of the first request having released + // the socket, so that it can be reused for the next request. + // This tests how HTTP connection reuse behaves. + setTimeout(next, 10); + }, + // Second put should work as well + next => put(testCase.bucketLoc, testCase.objLoc, 'localhost', 'obj2', next), + next => { + checkPut(testCase); + setTimeout(next, 10); + }, + // Overwriting PUT + next => put(testCase.bucketLoc, testCase.objLoc, 'localhost', 'obj2', next), + next => { + checkPut(testCase); + next(); + }, + ], + done, + ); }); }); }); diff --git a/tests/multipleBackend/objectPutCopyPart.js b/tests/multipleBackend/objectPutCopyPart.js index c319ca9df4..0cf3290295 100644 --- a/tests/multipleBackend/objectPutCopyPart.js +++ b/tests/multipleBackend/objectPutCopyPart.js @@ -1,18 +1,12 @@ const assert = require('assert'); const async = require('async'); const { parseString } = require('xml2js'); -const { - S3Client, - ListPartsCommand, - AbortMultipartUploadCommand, -} = require('@aws-sdk/client-s3'); +const { S3Client, ListPartsCommand, AbortMultipartUploadCommand } = require('@aws-sdk/client-s3'); const { storage, errors } = require('arsenal'); -const { cleanup, DummyRequestLogger, makeAuthInfo } - = require('../unit/helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../unit/helpers'); const { bucketPut } = require('../../lib/api/bucketPut'); -const initiateMultipartUpload - = require('../../lib/api/initiateMultipartUpload'); +const initiateMultipartUpload = require('../../lib/api/initiateMultipartUpload'); const objectPut = require('../../lib/api/objectPut'); const objectPutCopyPart = require('../../lib/api/objectPutCopyPart'); const DummyRequest = require('../unit/DummyRequest'); @@ -62,15 +56,16 @@ function getAwsParamsBucketMismatch(destObjName, uploadId) { return params; } -function copyPutPart(bucketLoc, mpuLoc, srcObjLoc, requestHost, cb, -errorPutCopyPart) { +function copyPutPart(bucketLoc, mpuLoc, srcObjLoc, requestHost, cb, errorPutCopyPart) { const keys = getSourceAndDestKeys(); const { sourceObjName, destObjName } = keys; - const post = bucketLoc ? '' + - '' + - `${bucketLoc}` + - '' : ''; + const post = bucketLoc + ? '' + + '' + + `${bucketLoc}` + + '' + : ''; const bucketPutReq = new DummyRequest({ bucketName, namespace, @@ -90,8 +85,10 @@ errorPutCopyPart) { actionImplicitDenies: false, }; if (mpuLoc) { - initiateReq.headers = { 'host': `${bucketName}.s3.amazonaws.com`, - 'x-amz-meta-scal-location-constraint': `${mpuLoc}` }; + initiateReq.headers = { + host: `${bucketName}.s3.amazonaws.com`, + 'x-amz-meta-scal-location-constraint': `${mpuLoc}`, + }; } if (requestHost) { initiateReq.parsedHost = requestHost; @@ -105,71 +102,78 @@ errorPutCopyPart) { actionImplicitDenies: false, }; if (srcObjLoc) { - sourceObjPutParams.headers = { 'host': `${bucketName}.s3.amazonaws.com`, - 'x-amz-meta-scal-location-constraint': `${srcObjLoc}` }; + sourceObjPutParams.headers = { + host: `${bucketName}.s3.amazonaws.com`, + 'x-amz-meta-scal-location-constraint': `${srcObjLoc}`, + }; } const sourceObjPutReq = new DummyRequest(sourceObjPutParams, body); if (requestHost) { sourceObjPutReq.parsedHost = requestHost; } - async.waterfall([ - next => { - bucketPut(authInfo, bucketPutReq, log, err => { - assert.ifError(err, 'Error putting bucket'); - next(err); - }); - }, - next => { - objectPut(authInfo, sourceObjPutReq, undefined, log, err => - next(err)); - }, - next => { - initiateMultipartUpload(authInfo, initiateReq, log, next); - }, - (result, corsHeaders, next) => { - const mpuKeys = metadata.keyMaps.get(mpuBucket); - assert.strictEqual(mpuKeys.size, 1); - assert(mpuKeys.keys().next().value - .startsWith(`overview${splitter}${destObjName}`)); - parseString(result, next); - }, - ], - (err, json) => { - // Need to build request in here since do not have - // uploadId until here - assert.ifError(err, 'Error putting source object or initiate MPU'); - const testUploadId = json.InitiateMultipartUploadResult. - UploadId[0]; - const copyPartParams = { - bucketName, - namespace, - objectKey: destObjName, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - url: `/${destObjName}?partNumber=1&uploadId=${testUploadId}`, - query: { - partNumber: '1', - uploadId: testUploadId, - }, - }; - const copyPartReq = new DummyRequest(copyPartParams); - return objectPutCopyPart(authInfo, copyPartReq, - bucketName, sourceObjName, undefined, log, (err, copyResult) => { - if (errorPutCopyPart) { - assert.strictEqual(err.code, errorPutCopyPart.statusCode); - assert(err.is[errorPutCopyPart.code]); - return cb(); - } - assert.strictEqual(err, null); - return parseString(copyResult, (err, json) => { - assert.equal(err, null, `Error parsing copy result ${err}`); - assert.strictEqual(json.CopyPartResult.ETag[0], - `"${partETag}"`); - assert(json.CopyPartResult.LastModified); - return cb(keys, testUploadId); + async.waterfall( + [ + next => { + bucketPut(authInfo, bucketPutReq, log, err => { + assert.ifError(err, 'Error putting bucket'); + next(err); }); - }); - }); + }, + next => { + objectPut(authInfo, sourceObjPutReq, undefined, log, err => next(err)); + }, + next => { + initiateMultipartUpload(authInfo, initiateReq, log, next); + }, + (result, corsHeaders, next) => { + const mpuKeys = metadata.keyMaps.get(mpuBucket); + assert.strictEqual(mpuKeys.size, 1); + assert(mpuKeys.keys().next().value.startsWith(`overview${splitter}${destObjName}`)); + parseString(result, next); + }, + ], + (err, json) => { + // Need to build request in here since do not have + // uploadId until here + assert.ifError(err, 'Error putting source object or initiate MPU'); + const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; + const copyPartParams = { + bucketName, + namespace, + objectKey: destObjName, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: `/${destObjName}?partNumber=1&uploadId=${testUploadId}`, + query: { + partNumber: '1', + uploadId: testUploadId, + }, + }; + const copyPartReq = new DummyRequest(copyPartParams); + return objectPutCopyPart( + authInfo, + copyPartReq, + bucketName, + sourceObjName, + undefined, + log, + (err, copyResult) => { + if (errorPutCopyPart) { + assert.strictEqual(err.code, errorPutCopyPart.statusCode); + assert(err.is[errorPutCopyPart.code]); + return cb(); + } + assert.strictEqual(err, null); + return parseString(copyResult, (err, json) => { + assert.equal(err, null, `Error parsing copy result ${err}`); + assert.strictEqual(json.CopyPartResult.ETag[0], `"${partETag}"`); + assert(json.CopyPartResult.LastModified); + return cb(keys, testUploadId); + }); + }, + ); + }, + ); } function assertPartList(partList, uploadId) { @@ -180,8 +184,7 @@ function assertPartList(partList, uploadId) { assert.strictEqual(partList.Parts[0].Size, 11); } -describeSkipIfE2E('ObjectCopyPutPart API with multiple backends', -function testSuite() { +describeSkipIfE2E('ObjectCopyPutPart API with multiple backends', function testSuite() { this.timeout(60000); beforeEach(() => { @@ -207,8 +210,7 @@ function testSuite() { }); it('should copy part to AWS based on mpu location', done => { - copyPutPart(memLocation, awsLocation, null, 'localhost', - (keys, uploadId) => { + copyPutPart(memLocation, awsLocation, null, 'localhost', (keys, uploadId) => { assert.strictEqual(ds.length, 2); const awsReq = getAwsParams(keys.destObjName, uploadId); s3.send(new ListPartsCommand(awsReq)) @@ -220,8 +222,10 @@ function testSuite() { done(); }) .catch(err => { - assert.fail(`Error with AWS operations: ${err}. ` + - `You may need to abort MPU with upload ID ${uploadId} manually.`); + assert.fail( + `Error with AWS operations: ${err}. ` + + `You may need to abort MPU with upload ID ${uploadId} manually.`, + ); }); }); }); @@ -266,63 +270,71 @@ function testSuite() { done(); }) .catch(err => { - assert.fail(`Error with AWS operations: ${err}. ` + - `You may need to abort MPU with upload ID ${uploadId} manually.`); + assert.fail( + `Error with AWS operations: ${err}. ` + + `You may need to abort MPU with upload ID ${uploadId} manually.`, + ); }); }); }); - it('should copy part an object on AWS location that has ' + - 'bucketMatch equals false to a mpu with a different AWS location', done => { - copyPutPart(null, awsLocation, awsLocationMismatch, 'localhost', - (keys, uploadId) => { - assert.deepStrictEqual(ds, []); - const awsReq = getAwsParams(keys.destObjName, uploadId); - s3.send(new ListPartsCommand(awsReq)) - .then(partList => { - assertPartList(partList, uploadId); - return s3.send(new AbortMultipartUploadCommand(awsReq)); - }) - .then(() => { - done(); - }) - .catch(err => { - assert.fail(`Error with AWS operations: ${err}. ` + - `You may need to abort MPU with upload ID ${uploadId} manually.`); - }); - }); - }); + it( + 'should copy part an object on AWS location that has ' + + 'bucketMatch equals false to a mpu with a different AWS location', + done => { + copyPutPart(null, awsLocation, awsLocationMismatch, 'localhost', (keys, uploadId) => { + assert.deepStrictEqual(ds, []); + const awsReq = getAwsParams(keys.destObjName, uploadId); + s3.send(new ListPartsCommand(awsReq)) + .then(partList => { + assertPartList(partList, uploadId); + return s3.send(new AbortMultipartUploadCommand(awsReq)); + }) + .then(() => { + done(); + }) + .catch(err => { + assert.fail( + `Error with AWS operations: ${err}. ` + + `You may need to abort MPU with upload ID ${uploadId} manually.`, + ); + }); + }); + }, + ); - it('should copy part an object on AWS to a mpu with a different ' + - 'AWS location that has bucketMatch equals false', done => { - copyPutPart(null, awsLocationMismatch, awsLocation, 'localhost', - (keys, uploadId) => { - assert.deepStrictEqual(ds, []); - const awsReq = getAwsParamsBucketMismatch(keys.destObjName, - uploadId); - s3.send(new ListPartsCommand(awsReq)) - .then(partList => { - assertPartList(partList, uploadId); - return s3.send(new AbortMultipartUploadCommand(awsReq)); - }) - .then(() => { - done(); - }) - .catch(err => { - assert.fail(`Error with AWS operations: ${err}. ` + - `You may need to abort MPU with upload ID ${uploadId} manually.`); - }); - }); - }); + it( + 'should copy part an object on AWS to a mpu with a different ' + + 'AWS location that has bucketMatch equals false', + done => { + copyPutPart(null, awsLocationMismatch, awsLocation, 'localhost', (keys, uploadId) => { + assert.deepStrictEqual(ds, []); + const awsReq = getAwsParamsBucketMismatch(keys.destObjName, uploadId); + s3.send(new ListPartsCommand(awsReq)) + .then(partList => { + assertPartList(partList, uploadId); + return s3.send(new AbortMultipartUploadCommand(awsReq)); + }) + .then(() => { + done(); + }) + .catch(err => { + assert.fail( + `Error with AWS operations: ${err}. ` + + `You may need to abort MPU with upload ID ${uploadId} manually.`, + ); + }); + }); + }, + ); // FIXME: does not pass, see CLDSRV-442 - it.skip('should return error 403 AccessDenied copying part to a ' + - 'different AWS location without object READ access', - done => { - copyPutPart(null, awsLocation, awsLocation2, 'localhost', done, - errors.AccessDenied); - }); - + it.skip( + 'should return error 403 AccessDenied copying part to a ' + 'different AWS location without object READ access', + done => { + copyPutPart(null, awsLocation, awsLocation2, 'localhost', done, errors.AccessDenied); + }, + ); it('should copy part to file based on request endpoint', done => { copyPutPart(null, null, memLocation, 'localhost', () => { diff --git a/tests/multipleBackend/objectPutPart.js b/tests/multipleBackend/objectPutPart.js index fcbf29ca1d..a87180a4a3 100644 --- a/tests/multipleBackend/objectPutPart.js +++ b/tests/multipleBackend/objectPutPart.js @@ -2,23 +2,18 @@ const assert = require('assert'); const async = require('async'); const crypto = require('crypto'); const { parseString } = require('xml2js'); -const { S3Client, - ListPartsCommand, - AbortMultipartUploadCommand } = require('@aws-sdk/client-s3'); +const { S3Client, ListPartsCommand, AbortMultipartUploadCommand } = require('@aws-sdk/client-s3'); const { storage } = require('arsenal'); const { config } = require('../../lib/Config'); -const { cleanup, DummyRequestLogger, makeAuthInfo } - = require('../unit/helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../unit/helpers'); const { bucketPut } = require('../../lib/api/bucketPut'); -const initiateMultipartUpload - = require('../../lib/api/initiateMultipartUpload'); +const initiateMultipartUpload = require('../../lib/api/initiateMultipartUpload'); const objectPutPart = require('../../lib/api/objectPutPart'); const DummyRequest = require('../unit/DummyRequest'); const mdWrapper = require('../../lib/metadata/wrapper'); const constants = require('../../constants'); -const { getRealAwsConfig } = - require('../functional/aws-node-sdk/test/support/awsConfig'); +const { getRealAwsConfig } = require('../functional/aws-node-sdk/test/support/awsConfig'); const { metadata } = storage.metadata.inMemory.metadata; const { ds } = storage.data.inMemory.datastore; @@ -51,14 +46,15 @@ function _getOverviewKey(objectKey, uploadId) { return `overview${splitter}${objectKey}${splitter}${uploadId}`; } -function putPart(bucketLoc, mpuLoc, requestHost, cb, -errorDescription) { +function putPart(bucketLoc, mpuLoc, requestHost, cb, errorDescription) { const objectName = `objectName-${Date.now()}`; - const post = bucketLoc ? '' + - '' + - `${bucketLoc}` + - '' : ''; + const post = bucketLoc + ? '' + + '' + + `${bucketLoc}` + + '' + : ''; const bucketPutReq = { bucketName, namespace, @@ -78,84 +74,88 @@ errorDescription) { actionImplicitDenies: false, }; if (mpuLoc) { - initiateReq.headers = { 'host': `${bucketName}.s3.amazonaws.com`, - 'x-amz-meta-scal-location-constraint': `${mpuLoc}` }; + initiateReq.headers = { + host: `${bucketName}.s3.amazonaws.com`, + 'x-amz-meta-scal-location-constraint': `${mpuLoc}`, + }; } if (requestHost) { initiateReq.parsedHost = requestHost; } - async.waterfall([ - next => { - bucketPut(authInfo, bucketPutReq, log, err => { - assert.ifError(err, 'Error putting bucket'); - next(err); - }); - }, - next => { - initiateMultipartUpload(authInfo, initiateReq, log, next); - }, - (result, corsHeaders, next) => { - const mpuKeys = metadata.keyMaps.get(mpuBucket); - assert.strictEqual(mpuKeys.size, 1); - assert(mpuKeys.keys().next().value - .startsWith(`overview${splitter}${objectName}`)); - parseString(result, next); - }, - ], - (err, json) => { - if (errorDescription) { - assert.strictEqual(err.code, 400); - assert(err.is.InvalidArgument); - assert(err.description.indexOf(errorDescription) > -1); - return cb(); - } - - assert.ifError(err, 'Error initiating MPU'); - - // Need to build request in here since do not have uploadId - // until here - const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; - const partReqParams = { - bucketName, - namespace, - objectKey: objectName, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - url: `/${objectName}?partNumber=1&uploadId=${testUploadId}`, - query: { - partNumber: '1', - uploadId: testUploadId, - }, - }; - const partReq = new DummyRequest(partReqParams, body1); - return objectPutPart(authInfo, partReq, undefined, log, err => { - assert.strictEqual(err, null); - if (bucketLoc !== awsLocation && mpuLoc !== awsLocation && - bucketLoc !== awsLocationMismatch && - mpuLoc !== awsLocationMismatch) { - const keysInMPUkeyMap = []; - metadata.keyMaps.get(mpuBucket).forEach((val, key) => { - keysInMPUkeyMap.push(key); - }); - const sortedKeyMap = keysInMPUkeyMap.sort(a => { - if (a.slice(0, 8) === 'overview') { - return -1; - } - return 0; + async.waterfall( + [ + next => { + bucketPut(authInfo, bucketPutReq, log, err => { + assert.ifError(err, 'Error putting bucket'); + next(err); }); - const partKey = sortedKeyMap[1]; - const partETag = metadata.keyMaps.get(mpuBucket) - .get(partKey)['content-md5']; - assert.strictEqual(keysInMPUkeyMap.length, 2); - assert.strictEqual(partETag, calculatedHash1); + }, + next => { + initiateMultipartUpload(authInfo, initiateReq, log, next); + }, + (result, corsHeaders, next) => { + const mpuKeys = metadata.keyMaps.get(mpuBucket); + assert.strictEqual(mpuKeys.size, 1); + assert(mpuKeys.keys().next().value.startsWith(`overview${splitter}${objectName}`)); + parseString(result, next); + }, + ], + (err, json) => { + if (errorDescription) { + assert.strictEqual(err.code, 400); + assert(err.is.InvalidArgument); + assert(err.description.indexOf(errorDescription) > -1); + return cb(); } - cb(objectName, testUploadId); - }); - }); + + assert.ifError(err, 'Error initiating MPU'); + + // Need to build request in here since do not have uploadId + // until here + const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; + const partReqParams = { + bucketName, + namespace, + objectKey: objectName, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: `/${objectName}?partNumber=1&uploadId=${testUploadId}`, + query: { + partNumber: '1', + uploadId: testUploadId, + }, + }; + const partReq = new DummyRequest(partReqParams, body1); + return objectPutPart(authInfo, partReq, undefined, log, err => { + assert.strictEqual(err, null); + if ( + bucketLoc !== awsLocation && + mpuLoc !== awsLocation && + bucketLoc !== awsLocationMismatch && + mpuLoc !== awsLocationMismatch + ) { + const keysInMPUkeyMap = []; + metadata.keyMaps.get(mpuBucket).forEach((val, key) => { + keysInMPUkeyMap.push(key); + }); + const sortedKeyMap = keysInMPUkeyMap.sort(a => { + if (a.slice(0, 8) === 'overview') { + return -1; + } + return 0; + }); + const partKey = sortedKeyMap[1]; + const partETag = metadata.keyMaps.get(mpuBucket).get(partKey)['content-md5']; + assert.strictEqual(keysInMPUkeyMap.length, 2); + assert.strictEqual(partETag, calculatedHash1); + } + cb(objectName, testUploadId); + }); + }, + ); } function listAndAbort(uploadId, calculatedHash2, objectName, location, done) { - const awsBucket = config.locationConstraints[location]. - details.bucketName; + const awsBucket = config.locationConstraints[location].details.bucketName; const params = { Bucket: awsBucket, Key: objectName, @@ -166,18 +166,22 @@ function listAndAbort(uploadId, calculatedHash2, objectName, location, done) { if (calculatedHash2) { assert.strictEqual(`"${calculatedHash2}"`, data.Parts[0].ETag); } - s3.send(new AbortMultipartUploadCommand(params)).then(() => { - done(); - }).catch(err => { - assert.equal(err, null, `Error aborting MPU: ${err}. ` + - `You must abort MPU with upload ID ${uploadId} manually.`); - done(); - }); + s3.send(new AbortMultipartUploadCommand(params)) + .then(() => { + done(); + }) + .catch(err => { + assert.equal( + err, + null, + `Error aborting MPU: ${err}. ` + `You must abort MPU with upload ID ${uploadId} manually.`, + ); + done(); + }); }); } -describeSkipIfE2E('objectPutPart API with multiple backends', -function testSuite() { +describeSkipIfE2E('objectPutPart API with multiple backends', function testSuite() { this.timeout(50000); beforeEach(() => { @@ -202,40 +206,35 @@ function testSuite() { }); it('should put a part to AWS based on mpu location', done => { - putPart(fileLocation, awsLocation, 'localhost', - (objectName, uploadId) => { + putPart(fileLocation, awsLocation, 'localhost', (objectName, uploadId) => { assert.deepStrictEqual(ds, []); listAndAbort(uploadId, null, objectName, awsLocation, done); }); }); - it('should replace part if two parts uploaded with same part number to AWS', - done => { - putPart(fileLocation, awsLocation, 'localhost', - (objectName, uploadId) => { + it('should replace part if two parts uploaded with same part number to AWS', done => { + putPart(fileLocation, awsLocation, 'localhost', (objectName, uploadId) => { assert.deepStrictEqual(ds, []); const partReqParams = { bucketName, namespace, objectKey: objectName, - headers: { 'host': `${bucketName}.s3.amazonaws.com`, - 'x-amz-meta-scal-location-constraint': awsLocation }, + headers: { host: `${bucketName}.s3.amazonaws.com`, 'x-amz-meta-scal-location-constraint': awsLocation }, url: `/${objectName}?partNumber=1&uploadId=${uploadId}`, query: { - partNumber: '1', uploadId, + partNumber: '1', + uploadId, }, }; const partReq = new DummyRequest(partReqParams, body2); objectPutPart(authInfo, partReq, undefined, log, err => { assert.equal(err, null, `Error putting second part: ${err}`); - listAndAbort(uploadId, calculatedHash2, - objectName, awsLocation, done); + listAndAbort(uploadId, calculatedHash2, objectName, awsLocation, done); }); }); }); - it('should upload part based on mpu location even if part ' + - 'location constraint is specified ', done => { + it('should upload part based on mpu location even if part ' + 'location constraint is specified ', done => { putPart(fileLocation, memLocation, 'localhost', () => { assert.deepStrictEqual(ds[1].value, body1); done(); @@ -257,29 +256,23 @@ function testSuite() { }); it('should put a part to AWS based on bucket location', done => { - putPart(awsLocation, null, 'localhost', - (objectName, uploadId) => { + putPart(awsLocation, null, 'localhost', (objectName, uploadId) => { assert.deepStrictEqual(ds, []); listAndAbort(uploadId, null, objectName, awsLocation, done); }); }); - it('should put a part to AWS based on bucket location with bucketMatch ' + - 'set to true', done => { - putPart(null, awsLocation, 'localhost', - (objectName, uploadId) => { + it('should put a part to AWS based on bucket location with bucketMatch ' + 'set to true', done => { + putPart(null, awsLocation, 'localhost', (objectName, uploadId) => { assert.deepStrictEqual(ds, []); listAndAbort(uploadId, null, objectName, awsLocation, done); }); }); - it('should put a part to AWS based on bucket location with bucketMatch ' + - 'set to false', done => { - putPart(null, awsLocationMismatch, 'localhost', - (objectName, uploadId) => { + it('should put a part to AWS based on bucket location with bucketMatch ' + 'set to false', done => { + putPart(null, awsLocationMismatch, 'localhost', (objectName, uploadId) => { assert.deepStrictEqual(ds, []); - listAndAbort(uploadId, null, `${bucketName}/${objectName}`, - awsLocationMismatch, done); + listAndAbort(uploadId, null, `${bucketName}/${objectName}`, awsLocationMismatch, done); }); }); @@ -290,28 +283,28 @@ function testSuite() { }); }); - it('should store a part even if the MPU was initiated on legacy version', - done => { - putPart('scality-internal-mem', null, 'localhost', - (objectKey, uploadId) => { + it('should store a part even if the MPU was initiated on legacy version', done => { + putPart('scality-internal-mem', null, 'localhost', (objectKey, uploadId) => { const mputOverviewKey = _getOverviewKey(objectKey, uploadId); - mdWrapper.getObjectMD(mpuBucket, mputOverviewKey, {}, log, - (err, res) => { + mdWrapper.getObjectMD(mpuBucket, mputOverviewKey, {}, log, (err, res) => { // remove location constraint to mimic legacy behvior // eslint-disable-next-line no-param-reassign res.controllingLocationConstraint = undefined; const md5Hash = crypto.createHash('md5'); const bufferBody = Buffer.from(body1); const calculatedHash = md5Hash.update(bufferBody).digest('hex'); - const partRequest = new DummyRequest({ - bucketName, - namespace, - objectKey, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - url: `/${objectKey}?partNumber=1&uploadId=${uploadId}`, - query: { partNumber: '1', uploadId }, - calculatedHash, - }, body1); + const partRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: `/${objectKey}?partNumber=1&uploadId=${uploadId}`, + query: { partNumber: '1', uploadId }, + calculatedHash, + }, + body1, + ); objectPutPart(authInfo, partRequest, undefined, log, err => { assert.strictEqual(err, null); const keysInMPUkeyMap = []; @@ -325,8 +318,7 @@ function testSuite() { return 0; }); const partKey = sortedKeyMap[1]; - const partETag = metadata.keyMaps.get(mpuBucket) - .get(partKey)['content-md5']; + const partETag = metadata.keyMaps.get(mpuBucket).get(partKey)['content-md5']; assert.strictEqual(keysInMPUkeyMap.length, 2); assert.strictEqual(partETag, calculatedHash); done(); diff --git a/tests/multipleBackend/routes/routeBackbeat.js b/tests/multipleBackend/routes/routeBackbeat.js index 7d47404522..e0eb42b65f 100644 --- a/tests/multipleBackend/routes/routeBackbeat.js +++ b/tests/multipleBackend/routes/routeBackbeat.js @@ -23,10 +23,7 @@ const versionIdUtils = versioning.VersionID; const { makeid } = require('../../unit/helpers'); const { makeRequest, makeBackbeatRequest } = require('../../functional/raw-node/utils/makeRequest'); const BucketUtility = require('../../functional/aws-node-sdk/lib/utility/bucket-util'); -const { - hasLocation, - describeSkipIfNotMultiple, -} = require('../../functional/aws-node-sdk/lib/utility/test-utils'); +const { hasLocation, describeSkipIfNotMultiple } = require('../../functional/aws-node-sdk/lib/utility/test-utils'); const { awsLocation, awsS3: awsClient, @@ -60,15 +57,12 @@ const testArn = 'aws::iam:123456789012:user/bart'; const testKey = 'testkey'; const testKeyUTF8 = '䆩鈁櫨㟔罳'; const testData = 'testkey data'; -const testDataMd5 = crypto.createHash('md5') - .update(testData, 'utf-8') - .digest('hex'); +const testDataMd5 = crypto.createHash('md5').update(testData, 'utf-8').digest('hex'); const emptyContentsMd5 = 'd41d8cd98f00b204e9800998ecf8427e'; const testMd = { 'md-model-version': 2, 'owner-display-name': 'Bart', - 'owner-id': ('79a59df900b949e55d96a1e698fbaced' + - 'fd6e09d98eacf8f8d5218e7cd47ef2be'), + 'owner-id': '79a59df900b949e55d96a1e698fbaced' + 'fd6e09d98eacf8f8d5218e7cd47ef2be', 'last-modified': '2017-05-15T20:32:40.032Z', 'content-length': testData.length, 'content-md5': testDataMd5, @@ -77,18 +71,18 @@ const testMd = { 'x-amz-server-side-encryption': '', 'x-amz-server-side-encryption-aws-kms-key-id': '', 'x-amz-server-side-encryption-customer-algorithm': '', - 'location': null, - 'acl': { + location: null, + acl: { Canned: 'private', FULL_CONTROL: [], WRITE_ACP: [], READ: [], READ_ACP: [], }, - 'nullVersionId': '99999999999999999999RG001 ', - 'isDeleteMarker': false, - 'versionId': '98505119639965999999RG001 ', - 'replicationInfo': { + nullVersionId: '99999999999999999999RG001 ', + isDeleteMarker: false, + versionId: '98505119639965999999RG001 ', + replicationInfo: { status: 'COMPLETED', backends: [{ site: 'zenko', status: 'PENDING' }], content: ['DATA', 'METADATA'], @@ -104,8 +98,7 @@ if (process.env.S3_TESTVAL_OWNERCANONICALID) { const nonVersionedTestMd = { 'owner-display-name': 'Bart', - 'owner-id': ('79a59df900b949e55d96a1e698fbaced' + - 'fd6e09d98eacf8f8d5218e7cd47ef2be'), + 'owner-id': '79a59df900b949e55d96a1e698fbaced' + 'fd6e09d98eacf8f8d5218e7cd47ef2be', 'content-length': testData.length, 'content-md5': testDataMd5, 'x-amz-version-id': 'null', @@ -114,19 +107,19 @@ const nonVersionedTestMd = { 'x-amz-server-side-encryption': '', 'x-amz-server-side-encryption-aws-kms-key-id': '', 'x-amz-server-side-encryption-customer-algorithm': '', - 'acl': { + acl: { Canned: 'private', FULL_CONTROL: [], WRITE_ACP: [], READ: [], READ_ACP: [], }, - 'location': null, - 'isNull': '', - 'nullVersionId': '', - 'isDeleteMarker': false, - 'tags': {}, - 'replicationInfo': { + location: null, + isNull: '', + nullVersionId: '', + isDeleteMarker: false, + tags: {}, + replicationInfo: { status: '', backends: [], content: [], @@ -137,40 +130,49 @@ const nonVersionedTestMd = { dataStoreVersionId: '', isNFS: null, }, - 'dataStoreName': 'us-east-1', + dataStoreName: 'us-east-1', 'last-modified': '2018-12-18T01:22:15.986Z', 'md-model-version': 3, }; function checkObjectData(s3, bucket, objectKey, dataValue, done) { - s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: objectKey, - })).then(async data => { - try { - const body = await data.Body.transformToString(); - assert.strictEqual(body, dataValue); - return done(); - } catch (err) { - return done(err); - } - }).catch(err => done(err)); + s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: objectKey, + }), + ) + .then(async data => { + try { + const body = await data.Body.transformToString(); + assert.strictEqual(body, dataValue); + return done(); + } catch (err) { + return done(err); + } + }) + .catch(err => done(err)); } function checkVersionData(s3, bucket, objectKey, versionId, dataValue, done) { - return s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: objectKey, - VersionId: versionId, - })).then(async data => { - try { - const body = await data.Body.transformToString(); - assert.strictEqual(body, dataValue); - return done(); - } catch (err) { - return done(err); - } - }).catch(err => done(err)); + return s3 + .send( + new GetObjectCommand({ + Bucket: bucket, + Key: objectKey, + VersionId: versionId, + }), + ) + .then(async data => { + try { + const body = await data.Body.transformToString(); + assert.strictEqual(body, dataValue); + return done(); + } catch (err) { + return done(err); + } + }) + .catch(err => done(err)); } function updateStorageClass(data, storageClass) { @@ -198,47 +200,63 @@ const itSkipS3C = process.env.S3_END_TO_END ? it.skip : it; describeSkipIfNotMultiple('backbeat DELETE routes', () => { itIfLocationAws('abort MPU', done => { const awsKey = 'backbeat-mpu-test'; - async.waterfall([ - next => { - awsClient.send(new CreateMultipartUploadCommand({ - Bucket: awsBucket, - Key: awsKey, - })).then(response => next(null, response)).catch(err => next(err)); - }, - (response, next) => { - const { UploadId } = response; - makeBackbeatRequest({ - method: 'DELETE', - bucket: awsBucket, - objectKey: awsKey, - resourceType: 'multiplebackenddata', - queryObj: { operation: 'abortmpu' }, - headers: { - 'x-scal-upload-id': UploadId, - 'x-scal-storage-type': 'aws_s3', - 'x-scal-storage-class': awsLocation, - }, - authCredentials: backbeatAuthCredentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - assert.deepStrictEqual(JSON.parse(response.body), {}); - return next(null, UploadId); - }); - }, (UploadId, next) => { - awsClient.send(new ListMultipartUploadsCommand({ - Bucket: awsBucket, - })).then(response => { - const hasOngoingUpload = - response.Uploads.some(upload => (upload === UploadId)); - assert(!hasOngoingUpload); - return next(); - }).catch(err => next(err)); + async.waterfall( + [ + next => { + awsClient + .send( + new CreateMultipartUploadCommand({ + Bucket: awsBucket, + Key: awsKey, + }), + ) + .then(response => next(null, response)) + .catch(err => next(err)); + }, + (response, next) => { + const { UploadId } = response; + makeBackbeatRequest( + { + method: 'DELETE', + bucket: awsBucket, + objectKey: awsKey, + resourceType: 'multiplebackenddata', + queryObj: { operation: 'abortmpu' }, + headers: { + 'x-scal-upload-id': UploadId, + 'x-scal-storage-type': 'aws_s3', + 'x-scal-storage-class': awsLocation, + }, + authCredentials: backbeatAuthCredentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + assert.deepStrictEqual(JSON.parse(response.body), {}); + return next(null, UploadId); + }, + ); + }, + (UploadId, next) => { + awsClient + .send( + new ListMultipartUploadsCommand({ + Bucket: awsBucket, + }), + ) + .then(response => { + const hasOngoingUpload = response.Uploads.some(upload => upload === UploadId); + assert(!hasOngoingUpload); + return next(); + }) + .catch(err => next(err)); + }, + ], + err => { + assert.ifError(err); + done(); }, - ], err => { - assert.ifError(err); - done(); - }); + ); }); }); @@ -246,13 +264,15 @@ function getMetadataToPut(putDataResponse) { const mdToPut = Object.assign({}, testMd); // Reproduce what backbeat does to update target metadata mdToPut.location = JSON.parse(putDataResponse.body); - ['x-amz-server-side-encryption', - 'x-amz-server-side-encryption-aws-kms-key-id', - 'x-amz-server-side-encryption-customer-algorithm'].forEach(headerName => { - if (putDataResponse.headers[headerName]) { - mdToPut[headerName] = putDataResponse.headers[headerName]; - } - }); + [ + 'x-amz-server-side-encryption', + 'x-amz-server-side-encryption-aws-kms-key-id', + 'x-amz-server-side-encryption-customer-algorithm', + ].forEach(headerName => { + if (putDataResponse.headers[headerName]) { + mdToPut[headerName] = putDataResponse.headers[headerName]; + } + }); return mdToPut; } @@ -269,40 +289,50 @@ describe('backbeat routes', () => { before(done => { bucketUtil = new BucketUtility('default', {}); s3 = bucketUtil.s3; - bucketUtil.emptyManyIfExists([TEST_BUCKET, TEST_ENCRYPTED_BUCKET, NONVERSIONED_BUCKET, - VERSION_SUSPENDED_BUCKET]) + bucketUtil + .emptyManyIfExists([TEST_BUCKET, TEST_ENCRYPTED_BUCKET, NONVERSIONED_BUCKET, VERSION_SUSPENDED_BUCKET]) .then(async () => { try { await s3.send(new CreateBucketCommand({ Bucket: TEST_BUCKET })); - await s3.send(new PutBucketVersioningCommand({ - Bucket: TEST_BUCKET, - VersioningConfiguration: { Status: 'Enabled' }, - })); - await s3.send(new CreateBucketCommand({ - Bucket: NONVERSIONED_BUCKET, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: TEST_BUCKET, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ); + await s3.send( + new CreateBucketCommand({ + Bucket: NONVERSIONED_BUCKET, + }), + ); await s3.send(new CreateBucketCommand({ Bucket: VERSION_SUSPENDED_BUCKET })); - await s3.send(new PutBucketVersioningCommand({ - Bucket: VERSION_SUSPENDED_BUCKET, - VersioningConfiguration: { Status: 'Suspended' }, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: VERSION_SUSPENDED_BUCKET, + VersioningConfiguration: { Status: 'Suspended' }, + }), + ); await s3.send(new CreateBucketCommand({ Bucket: TEST_ENCRYPTED_BUCKET })); - await s3.send(new PutBucketVersioningCommand({ - Bucket: TEST_ENCRYPTED_BUCKET, - VersioningConfiguration: { Status: 'Enabled' }, - })); - await s3.send(new PutBucketEncryptionCommand({ - Bucket: TEST_ENCRYPTED_BUCKET, - ServerSideEncryptionConfiguration: { - Rules: [ - { - ApplyServerSideEncryptionByDefault: { - SSEAlgorithm: 'AES256', + await s3.send( + new PutBucketVersioningCommand({ + Bucket: TEST_ENCRYPTED_BUCKET, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ); + await s3.send( + new PutBucketEncryptionCommand({ + Bucket: TEST_ENCRYPTED_BUCKET, + ServerSideEncryptionConfiguration: { + Rules: [ + { + ApplyServerSideEncryptionByDefault: { + SSEAlgorithm: 'AES256', + }, }, - }, - ], - }, - })); + ], + }, + }), + ); done(); } catch (err) { done(err); @@ -315,14 +345,14 @@ describe('backbeat routes', () => { }); after(async () => { - await bucketUtil.empty(TEST_BUCKET); - await s3.send(new DeleteBucketCommand({ Bucket: TEST_BUCKET })); - await bucketUtil.empty(TEST_ENCRYPTED_BUCKET); - await s3.send(new DeleteBucketCommand({ Bucket: TEST_ENCRYPTED_BUCKET })); - await bucketUtil.empty(NONVERSIONED_BUCKET); - await s3.send(new DeleteBucketCommand({ Bucket: NONVERSIONED_BUCKET })); - await bucketUtil.empty(VERSION_SUSPENDED_BUCKET); - await s3.send(new DeleteBucketCommand({ Bucket: VERSION_SUSPENDED_BUCKET })); + await bucketUtil.empty(TEST_BUCKET); + await s3.send(new DeleteBucketCommand({ Bucket: TEST_BUCKET })); + await bucketUtil.empty(TEST_ENCRYPTED_BUCKET); + await s3.send(new DeleteBucketCommand({ Bucket: TEST_ENCRYPTED_BUCKET })); + await bucketUtil.empty(NONVERSIONED_BUCKET); + await s3.send(new DeleteBucketCommand({ Bucket: NONVERSIONED_BUCKET })); + await bucketUtil.empty(VERSION_SUSPENDED_BUCKET); + await s3.send(new DeleteBucketCommand({ Bucket: VERSION_SUSPENDED_BUCKET })); }); describe('null version', () => { @@ -346,2034 +376,2813 @@ describe('backbeat routes', () => { beforeEach(() => { bucket = generateUniqueBucketName(BUCKET_FOR_NULL_VERSION_PREFIX); - return bucketUtil.emptyIfExists(bucket) - .then(() => s3.send(new CreateBucketCommand({ Bucket: bucket }))); + return bucketUtil.emptyIfExists(bucket).then(() => s3.send(new CreateBucketCommand({ Bucket: bucket }))); }); - afterEach(() => bucketUtil.empty(bucket) - .then(() => s3.send(new DeleteBucketCommand({ Bucket: bucket }))) - ); + afterEach(() => bucketUtil.empty(bucket).then(() => s3.send(new DeleteBucketCommand({ Bucket: bucket })))); it('should update metadata of a current null version', done => { let objMD; - async.series({ - putObject: next => { - s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: keyName, - Body: Buffer.from(testData), - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }); - }, - enableVersioningSource: next => { - s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Enabled' }, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }); - }, - getMetadata: next => makeBackbeatRequest({ - method: 'GET', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', + async.series( + { + putObject: next => { + s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: keyName, + Body: Buffer.from(testData), + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }); }, - authCredentials: backbeatAuthCredentials, - }, (err, data) => { + enableVersioningSource: next => { + s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }); + }, + getMetadata: next => + makeBackbeatRequest( + { + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + }, + (err, data) => { + if (err) { + return next(err); + } + const { error, result } = updateStorageClass(data, storageClass); + if (error) { + return next(error); + } + objMD = result; + return next(); + }, + ), + putMetadata: next => + makeBackbeatRequest( + { + method: 'PUT', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + requestBody: objMD, + }, + next, + ), + headObject: next => + s3 + .send( + new HeadObjectCommand({ + Bucket: bucket, + Key: keyName, + VersionId: 'null', + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + getMetadataAfter: next => + makeBackbeatRequest( + { + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + }, + next, + ), + listObjectVersions: next => + s3 + .send( + new ListObjectVersionsCommand({ + Bucket: bucket, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + }, + (err, results) => { if (err) { - return next(err); + return done(err); } - const { error, result } = updateStorageClass(data, storageClass); - if (error) { - return next(error); + const headObjectRes = results.headObject; + assert.strictEqual(headObjectRes.VersionId, 'null'); + assert.strictEqual(headObjectRes.StorageClass, storageClass); + + const getMetadataAfterRes = results.getMetadataAfter; + const objMDAfter = JSON.parse(getMetadataAfterRes.body).Body; + const expectedMd = JSON.parse(objMD); + expectedMd.isNull = true; // TODO remove the line once CLDSRV-509 is fixed + if (!isNullVersionCompatMode) { + expectedMd.isNull2 = true; // TODO remove the line once CLDSRV-509 is fixed } - objMD = result; - return next(); - }), - putMetadata: next => makeBackbeatRequest({ - method: 'PUT', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', - }, - authCredentials: backbeatAuthCredentials, - requestBody: objMD, - }, next), - headObject: next => s3.send(new HeadObjectCommand({ - Bucket: bucket, - Key: keyName, - VersionId: 'null', - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - getMetadataAfter: next => makeBackbeatRequest({ - method: 'GET', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', - }, - authCredentials: backbeatAuthCredentials, - }, next), - listObjectVersions: next => s3.send(new ListObjectVersionsCommand({ - Bucket: bucket, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - }, (err, results) => { - if (err) { - return done(err); - } - const headObjectRes = results.headObject; - assert.strictEqual(headObjectRes.VersionId, 'null'); - assert.strictEqual(headObjectRes.StorageClass, storageClass); - - const getMetadataAfterRes = results.getMetadataAfter; - const objMDAfter = JSON.parse(getMetadataAfterRes.body).Body; - const expectedMd = JSON.parse(objMD); - expectedMd.isNull = true; // TODO remove the line once CLDSRV-509 is fixed - if (!isNullVersionCompatMode) { - expectedMd.isNull2 = true; // TODO remove the line once CLDSRV-509 is fixed - } - assert.deepStrictEqual(JSON.parse(objMDAfter), expectedMd); + assert.deepStrictEqual(JSON.parse(objMDAfter), expectedMd); - const listObjectVersionsRes = results.listObjectVersions; - const { Versions } = listObjectVersionsRes; + const listObjectVersionsRes = results.listObjectVersions; + const { Versions } = listObjectVersionsRes; - assert.strictEqual(Versions.length, 1); + assert.strictEqual(Versions.length, 1); - const [currentVersion] = Versions; - assertVersionIsNullAndUpdated(currentVersion); - return done(); - }); + const [currentVersion] = Versions; + assertVersionIsNullAndUpdated(currentVersion); + return done(); + }, + ); }); it('should update metadata of a non-current null version', done => { let objMD; let expectedVersionId; - return async.series({ - putObjectInitial: next => { - s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: keyName, - Body: Buffer.from(testData), - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }); - }, - enableVersioning: next => { - s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Enabled' }, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }); - }, - putObjectAgain: next => { - s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: keyName, - Body: Buffer.from(testData), - })).then(data => { - expectedVersionId = data.VersionId; - return next(null, data); - }).catch(err => { - next(err); - }); - }, - getMetadata: next => makeBackbeatRequest({ - method: 'GET', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', + return async.series( + { + putObjectInitial: next => { + s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: keyName, + Body: Buffer.from(testData), + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }); }, - authCredentials: backbeatAuthCredentials, - }, (err, data) => { - if (err) { - return next(err); - } - const { error, result } = updateStorageClass(data, storageClass); - if (error) { - return next(error); - } - objMD = result; - return next(); - }), - putMetadata: next => makeBackbeatRequest({ - method: 'PUT', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', + enableVersioning: next => { + s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }); }, - authCredentials: backbeatAuthCredentials, - requestBody: objMD, - }, next), - headObject: next => { - s3.send(new HeadObjectCommand({ - Bucket: bucket, - Key: keyName, - VersionId: 'null', - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }); - }, - getMetadataAfter: next => makeBackbeatRequest({ - method: 'GET', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', + putObjectAgain: next => { + s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: keyName, + Body: Buffer.from(testData), + }), + ) + .then(data => { + expectedVersionId = data.VersionId; + return next(null, data); + }) + .catch(err => { + next(err); + }); + }, + getMetadata: next => + makeBackbeatRequest( + { + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + }, + (err, data) => { + if (err) { + return next(err); + } + const { error, result } = updateStorageClass(data, storageClass); + if (error) { + return next(error); + } + objMD = result; + return next(); + }, + ), + putMetadata: next => + makeBackbeatRequest( + { + method: 'PUT', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + requestBody: objMD, + }, + next, + ), + headObject: next => { + s3.send( + new HeadObjectCommand({ + Bucket: bucket, + Key: keyName, + VersionId: 'null', + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }); + }, + getMetadataAfter: next => + makeBackbeatRequest( + { + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + }, + next, + ), + listObjectVersions: next => { + s3.send( + new ListObjectVersionsCommand({ + Bucket: bucket, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }); }, - authCredentials: backbeatAuthCredentials, - }, next), - listObjectVersions: next => { - s3.send(new ListObjectVersionsCommand({ - Bucket: bucket, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }); }, - }, (err, results) => { - if (err) { - return done(err); - } - const headObjectRes = results.headObject; - assert.strictEqual(headObjectRes.VersionId, 'null'); - assert.strictEqual(headObjectRes.StorageClass, storageClass); + (err, results) => { + if (err) { + return done(err); + } + const headObjectRes = results.headObject; + assert.strictEqual(headObjectRes.VersionId, 'null'); + assert.strictEqual(headObjectRes.StorageClass, storageClass); - const getMetadataAfterRes = results.getMetadataAfter; - const objMDAfter = JSON.parse(getMetadataAfterRes.body).Body; - assert.deepStrictEqual(JSON.parse(objMDAfter), JSON.parse(objMD)); + const getMetadataAfterRes = results.getMetadataAfter; + const objMDAfter = JSON.parse(getMetadataAfterRes.body).Body; + assert.deepStrictEqual(JSON.parse(objMDAfter), JSON.parse(objMD)); - const listObjectVersionsRes = results.listObjectVersions; - const { Versions } = listObjectVersionsRes; + const listObjectVersionsRes = results.listObjectVersions; + const { Versions } = listObjectVersionsRes; - assert.strictEqual(Versions.length, 2); - const currentVersion = Versions.find(v => v.IsLatest); - assertVersionHasNotBeenUpdated(currentVersion, expectedVersionId); + assert.strictEqual(Versions.length, 2); + const currentVersion = Versions.find(v => v.IsLatest); + assertVersionHasNotBeenUpdated(currentVersion, expectedVersionId); - const [nonCurrentVersion] = Versions.filter(v => !v.IsLatest); - assertVersionIsNullAndUpdated(nonCurrentVersion); - return done(); - }); + const [nonCurrentVersion] = Versions.filter(v => !v.IsLatest); + assertVersionIsNullAndUpdated(nonCurrentVersion); + return done(); + }, + ); }); it('should update metadata of a suspended null version', done => { let objMD; - return async.series({ - suspendVersioning: next => { - s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Suspended' }, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }); - }, - putObject: next => { - s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: keyName, - Body: Buffer.from(testData), - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }); - }, - enableVersioning: next => { - s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Enabled' }, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }); - }, - getMetadata: next => makeBackbeatRequest({ - method: 'GET', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', + return async.series( + { + suspendVersioning: next => { + s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Suspended' }, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }); }, - authCredentials: backbeatAuthCredentials, - }, (err, data) => { - if (err) { - return next(err); - } - const { error, result } = updateStorageClass(data, storageClass); - if (error) { - return next(error); - } - objMD = result; - return next(); - }), - putUpdatedMetadata: next => makeBackbeatRequest({ - method: 'PUT', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', + putObject: next => { + s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: keyName, + Body: Buffer.from(testData), + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }); }, - authCredentials: backbeatAuthCredentials, - requestBody: objMD, - }, next), - headObject: next => { - s3.send(new HeadObjectCommand({ - Bucket: bucket, - Key: keyName, - VersionId: 'null', - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }); - }, - getMetadataAfter: next => makeBackbeatRequest({ - method: 'GET', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', + enableVersioning: next => { + s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }); + }, + getMetadata: next => + makeBackbeatRequest( + { + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + }, + (err, data) => { + if (err) { + return next(err); + } + const { error, result } = updateStorageClass(data, storageClass); + if (error) { + return next(error); + } + objMD = result; + return next(); + }, + ), + putUpdatedMetadata: next => + makeBackbeatRequest( + { + method: 'PUT', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + requestBody: objMD, + }, + next, + ), + headObject: next => { + s3.send( + new HeadObjectCommand({ + Bucket: bucket, + Key: keyName, + VersionId: 'null', + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }); + }, + getMetadataAfter: next => + makeBackbeatRequest( + { + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + }, + next, + ), + listObjectVersions: next => { + s3.send( + new ListObjectVersionsCommand({ + Bucket: bucket, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }); }, - authCredentials: backbeatAuthCredentials, - }, next), - listObjectVersions: next => { - s3.send(new ListObjectVersionsCommand({ - Bucket: bucket, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }); }, - }, (err, results) => { - if (err) { - return done(err); - } - const headObjectRes = results.headObject; - assert.strictEqual(headObjectRes.VersionId, 'null'); - assert.strictEqual(headObjectRes.StorageClass, storageClass); + (err, results) => { + if (err) { + return done(err); + } + const headObjectRes = results.headObject; + assert.strictEqual(headObjectRes.VersionId, 'null'); + assert.strictEqual(headObjectRes.StorageClass, storageClass); - const getMetadataAfterRes = results.getMetadataAfter; - const objMDAfter = JSON.parse(getMetadataAfterRes.body).Body; - assert.deepStrictEqual(JSON.parse(objMDAfter), JSON.parse(objMD)); + const getMetadataAfterRes = results.getMetadataAfter; + const objMDAfter = JSON.parse(getMetadataAfterRes.body).Body; + assert.deepStrictEqual(JSON.parse(objMDAfter), JSON.parse(objMD)); - const listObjectVersionsRes = results.listObjectVersions; - const { Versions } = listObjectVersionsRes; + const listObjectVersionsRes = results.listObjectVersions; + const { Versions } = listObjectVersionsRes; - assert.strictEqual(Versions.length, 1); + assert.strictEqual(Versions.length, 1); - const [currentVersion] = Versions; - assertVersionIsNullAndUpdated(currentVersion); - return done(); - }); + const [currentVersion] = Versions; + assertVersionIsNullAndUpdated(currentVersion); + return done(); + }, + ); }); it('should update metadata of a suspended null version with internal version id', done => { let objMD; - return async.series({ - suspendVersioning: next => { - s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Suspended' }, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }); - }, - putObject: next => { - s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: keyName, - Body: Buffer.from(testData), - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }); - }, - enableVersioning: next => { - s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Enabled' }, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }); - }, - putObjectTagging: next => { - s3.send(new PutObjectTaggingCommand({ - Bucket: bucket, - Key: keyName, - VersionId: 'null', - Tagging: { TagSet: [{ Key: 'key1', Value: 'value1' }] }, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }); - }, - getMetadata: next => makeBackbeatRequest({ - method: 'GET', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', + return async.series( + { + suspendVersioning: next => { + s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Suspended' }, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }); }, - authCredentials: backbeatAuthCredentials, - }, (err, data) => { - if (err) { - return next(err); - } - const { error, result } = updateStorageClass(data, storageClass); - if (error) { - return next(error); - } - objMD = result; - return next(); - }), - putUpdatedMetadata: next => makeBackbeatRequest({ - method: 'PUT', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', + putObject: next => { + s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: keyName, + Body: Buffer.from(testData), + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }); }, - authCredentials: backbeatAuthCredentials, - requestBody: objMD, - }, next), - headObject: next => { - s3.send(new HeadObjectCommand({ - Bucket: bucket, - Key: keyName, - VersionId: 'null', - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }); - }, - getMetadataAfter: next => makeBackbeatRequest({ - method: 'GET', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', + enableVersioning: next => { + s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }); + }, + putObjectTagging: next => { + s3.send( + new PutObjectTaggingCommand({ + Bucket: bucket, + Key: keyName, + VersionId: 'null', + Tagging: { TagSet: [{ Key: 'key1', Value: 'value1' }] }, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }); + }, + getMetadata: next => + makeBackbeatRequest( + { + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + }, + (err, data) => { + if (err) { + return next(err); + } + const { error, result } = updateStorageClass(data, storageClass); + if (error) { + return next(error); + } + objMD = result; + return next(); + }, + ), + putUpdatedMetadata: next => + makeBackbeatRequest( + { + method: 'PUT', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + requestBody: objMD, + }, + next, + ), + headObject: next => { + s3.send( + new HeadObjectCommand({ + Bucket: bucket, + Key: keyName, + VersionId: 'null', + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }); + }, + getMetadataAfter: next => + makeBackbeatRequest( + { + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + }, + next, + ), + listObjectVersions: next => { + s3.send( + new ListObjectVersionsCommand({ + Bucket: bucket, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }); }, - authCredentials: backbeatAuthCredentials, - }, next), - listObjectVersions: next => { - s3.send(new ListObjectVersionsCommand({ - Bucket: bucket, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }); }, - }, (err, results) => { - if (err) { - return done(err); - } - const headObjectRes = results.headObject; - assert.strictEqual(headObjectRes.VersionId, 'null'); - assert.strictEqual(headObjectRes.StorageClass, storageClass); + (err, results) => { + if (err) { + return done(err); + } + const headObjectRes = results.headObject; + assert.strictEqual(headObjectRes.VersionId, 'null'); + assert.strictEqual(headObjectRes.StorageClass, storageClass); - const getMetadataAfterRes = results.getMetadataAfter; - const objMDAfter = JSON.parse(getMetadataAfterRes.body).Body; - assert.deepStrictEqual(JSON.parse(objMDAfter), JSON.parse(objMD)); + const getMetadataAfterRes = results.getMetadataAfter; + const objMDAfter = JSON.parse(getMetadataAfterRes.body).Body; + assert.deepStrictEqual(JSON.parse(objMDAfter), JSON.parse(objMD)); - const listObjectVersionsRes = results.listObjectVersions; - const { Versions } = listObjectVersionsRes; + const listObjectVersionsRes = results.listObjectVersions; + const { Versions } = listObjectVersionsRes; - assert.strictEqual(Versions.length, 1); + assert.strictEqual(Versions.length, 1); - const currentVersion = Versions[0]; - assert(currentVersion.IsLatest); - assertVersionIsNullAndUpdated(currentVersion); - return done(); - }); + const currentVersion = Versions[0]; + assert(currentVersion.IsLatest); + assertVersionIsNullAndUpdated(currentVersion); + return done(); + }, + ); }); it('should update metadata of a non-version object', done => { let objMD; - async.series([ - next => s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: keyName, - Body: Buffer.from(testData), - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => makeBackbeatRequest({ - method: 'GET', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', - }, - authCredentials: backbeatAuthCredentials, - }, (err, data) => { + async.series( + [ + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: keyName, + Body: Buffer.from(testData), + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + makeBackbeatRequest( + { + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + }, + (err, data) => { + if (err) { + return next(err); + } + const { error, result } = updateStorageClass(data, storageClass); + if (error) { + return next(error); + } + objMD = result; + return next(); + }, + ), + next => + makeBackbeatRequest( + { + method: 'PUT', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + requestBody: objMD, + }, + next, + ), + next => + s3 + .send( + new HeadObjectCommand({ + Bucket: bucket, + Key: keyName, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new ListObjectVersionsCommand({ + Bucket: bucket, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + ], + (err, data) => { if (err) { - return next(err); - } - const { error, result } = updateStorageClass(data, storageClass); - if (error) { - return next(error); + return done(err); } - objMD = result; - return next(); - }), - next => makeBackbeatRequest({ - method: 'PUT', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', - }, - authCredentials: backbeatAuthCredentials, - requestBody: objMD, - }, next), - next => s3.send(new HeadObjectCommand({ - Bucket: bucket, - Key: keyName, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new ListObjectVersionsCommand({ - Bucket: bucket, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - ], (err, data) => { - if (err) { - return done(err); - } - const headObjectRes = data[3]; - assert(!headObjectRes.VersionId); - assert.strictEqual(headObjectRes.StorageClass, storageClass); + const headObjectRes = data[3]; + assert(!headObjectRes.VersionId); + assert.strictEqual(headObjectRes.StorageClass, storageClass); - const listObjectVersionsRes = data[4]; - const { DeleteMarkers, Versions } = listObjectVersionsRes; + const listObjectVersionsRes = data[4]; + const { DeleteMarkers, Versions } = listObjectVersionsRes; - assert.strictEqual(DeleteMarkers, undefined); - assert.strictEqual(Versions.length, 1); + assert.strictEqual(DeleteMarkers, undefined); + assert.strictEqual(Versions.length, 1); - const currentVersion = Versions[0]; - assert(currentVersion.IsLatest); - assertVersionIsNullAndUpdated(currentVersion); - return done(); - }); + const currentVersion = Versions[0]; + assert(currentVersion.IsLatest); + assertVersionIsNullAndUpdated(currentVersion); + return done(); + }, + ); }); it('should create a new null version if versioning suspended and no version', done => { let objMD; - async.series([ - next => { - s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Suspended' }, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }); - }, - next => { - s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: keyName, - Body: Buffer.from(testData), - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }); - }, - next => makeBackbeatRequest({ - method: 'GET', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', + async.series( + [ + next => { + s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Suspended' }, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }); }, - authCredentials: backbeatAuthCredentials, - }, (err, data) => { - if (err) { - return next(err); - } - const { error, result } = updateStorageClass(data, storageClass); - if (error) { - return next(error); - } - objMD = result; - return next(); - }), - next => { - s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: keyName, - VersionId: 'null', - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }); - }, - next => makeBackbeatRequest({ - method: 'PUT', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', + next => { + s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: keyName, + Body: Buffer.from(testData), + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }); }, - authCredentials: backbeatAuthCredentials, - requestBody: objMD, - }, next), - next => { - s3.send(new HeadObjectCommand({ - Bucket: bucket, - Key: keyName, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }); - }, - next => { - s3.send(new ListObjectVersionsCommand({ - Bucket: bucket - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }); - }, - ], (err, data) => { - if (err) { - return done(err); - } - const headObjectRes = data[5]; - assert.strictEqual(headObjectRes.VersionId, 'null'); - assert.strictEqual(headObjectRes.StorageClass, storageClass); + next => + makeBackbeatRequest( + { + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + }, + (err, data) => { + if (err) { + return next(err); + } + const { error, result } = updateStorageClass(data, storageClass); + if (error) { + return next(error); + } + objMD = result; + return next(); + }, + ), + next => { + s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: keyName, + VersionId: 'null', + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }); + }, + next => + makeBackbeatRequest( + { + method: 'PUT', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + requestBody: objMD, + }, + next, + ), + next => { + s3.send( + new HeadObjectCommand({ + Bucket: bucket, + Key: keyName, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }); + }, + next => { + s3.send( + new ListObjectVersionsCommand({ + Bucket: bucket, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }); + }, + ], + (err, data) => { + if (err) { + return done(err); + } + const headObjectRes = data[5]; + assert.strictEqual(headObjectRes.VersionId, 'null'); + assert.strictEqual(headObjectRes.StorageClass, storageClass); - const listObjectVersionsRes = data[6]; - const { DeleteMarkers, Versions } = listObjectVersionsRes; + const listObjectVersionsRes = data[6]; + const { DeleteMarkers, Versions } = listObjectVersionsRes; - assert.strictEqual(DeleteMarkers, undefined); - assert.strictEqual(Versions.length, 1); + assert.strictEqual(DeleteMarkers, undefined); + assert.strictEqual(Versions.length, 1); - const currentVersion = Versions[0]; - assert(currentVersion.IsLatest); + const currentVersion = Versions[0]; + assert(currentVersion.IsLatest); - assertVersionIsNullAndUpdated(currentVersion); + assertVersionIsNullAndUpdated(currentVersion); - return done(); - }); + return done(); + }, + ); }); // TODO fix broken on S3C with metadata backend,create 2 null Versions itSkipS3C('should create a new null version if versioning suspended and delete marker null version', done => { let objMD; - return async.series([ - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Suspended' }, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: keyName, - Body: Buffer.from(testData), - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => makeBackbeatRequest({ - method: 'GET', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', - }, - authCredentials: backbeatAuthCredentials, - }, (err, data) => { + return async.series( + [ + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Suspended' }, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: keyName, + Body: Buffer.from(testData), + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + makeBackbeatRequest( + { + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + }, + (err, data) => { + if (err) { + return next(err); + } + const { error, result } = updateStorageClass(data, storageClass); + if (error) { + return next(error); + } + objMD = result; + return next(); + }, + ), + next => + s3 + .send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: keyName, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + makeBackbeatRequest( + { + method: 'PUT', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + requestBody: objMD, + }, + next, + ), + next => + s3 + .send( + new HeadObjectCommand({ + Bucket: bucket, + Key: keyName, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new ListObjectVersionsCommand({ + Bucket: bucket, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + ], + (err, data) => { if (err) { - return next(err); + return done(err); } - const { error, result } = updateStorageClass(data, storageClass); - if (error) { - return next(error); - } - objMD = result; - return next(); - }), - next => s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: keyName, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => makeBackbeatRequest({ - method: 'PUT', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', - }, - authCredentials: backbeatAuthCredentials, - requestBody: objMD, - }, next), - next => s3.send(new HeadObjectCommand({ - Bucket: bucket, - Key: keyName, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new ListObjectVersionsCommand({ - Bucket: bucket - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - ], (err, data) => { - if (err) { - return done(err); - } - const headObjectRes = data[5]; - assert.strictEqual(headObjectRes.VersionId, 'null'); - assert.strictEqual(headObjectRes.StorageClass, storageClass); + const headObjectRes = data[5]; + assert.strictEqual(headObjectRes.VersionId, 'null'); + assert.strictEqual(headObjectRes.StorageClass, storageClass); - const listObjectVersionsRes = data[6]; - const { DeleteMarkers, Versions } = listObjectVersionsRes; + const listObjectVersionsRes = data[6]; + const { DeleteMarkers, Versions } = listObjectVersionsRes; - assert.strictEqual(DeleteMarkers, undefined); - assert.strictEqual(Versions.length, 1); + assert.strictEqual(DeleteMarkers, undefined); + assert.strictEqual(Versions.length, 1); - const currentVersion = Versions[0]; - assert(currentVersion.IsLatest); - assertVersionIsNullAndUpdated(currentVersion); - return done(); - }); + const currentVersion = Versions[0]; + assert(currentVersion.IsLatest); + assertVersionIsNullAndUpdated(currentVersion); + return done(); + }, + ); }); it('should create a new null version if versioning suspended and version has version id', done => { let expectedVersionId; let objMD; - return async.series([ - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Enabled' }, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: keyName, - Body: Buffer.from(testData), - })).then(data => { - expectedVersionId = data.VersionId; - return next(null, data); - }).catch(err => { - next(err); - }), - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Suspended' }, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: keyName, - Body: Buffer.from(testData), - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => makeBackbeatRequest({ - method: 'GET', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: null, - }, - authCredentials: backbeatAuthCredentials, - }, (err, data) => { + return async.series( + [ + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: keyName, + Body: Buffer.from(testData), + }), + ) + .then(data => { + expectedVersionId = data.VersionId; + return next(null, data); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Suspended' }, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: keyName, + Body: Buffer.from(testData), + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + makeBackbeatRequest( + { + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: null, + }, + authCredentials: backbeatAuthCredentials, + }, + (err, data) => { + if (err) { + return next(err); + } + const { error, result } = updateStorageClass(data, storageClass); + if (error) { + return next(error); + } + objMD = result; + return next(); + }, + ), + next => + s3 + .send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: keyName, + VersionId: 'null', + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + makeBackbeatRequest( + { + method: 'PUT', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + requestBody: objMD, + }, + next, + ), + next => + s3 + .send( + new HeadObjectCommand({ + Bucket: bucket, + Key: keyName, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send(new ListObjectVersionsCommand({ Bucket: bucket })) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + ], + (err, data) => { if (err) { - return next(err); - } - const { error, result } = updateStorageClass(data, storageClass); - if (error) { - return next(error); + return done(err); } - objMD = result; - return next(); - }), - next => s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: keyName, - VersionId: 'null', - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => makeBackbeatRequest({ - method: 'PUT', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', - }, - authCredentials: backbeatAuthCredentials, - requestBody: objMD, - }, next), - next => s3.send(new HeadObjectCommand({ - Bucket: bucket, - Key: keyName, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new ListObjectVersionsCommand({ Bucket: bucket })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - ], (err, data) => { - if (err) { - return done(err); - } - const headObjectRes = data[7]; - assert.strictEqual(headObjectRes.VersionId, 'null'); - assert.strictEqual(headObjectRes.StorageClass, storageClass); + const headObjectRes = data[7]; + assert.strictEqual(headObjectRes.VersionId, 'null'); + assert.strictEqual(headObjectRes.StorageClass, storageClass); - const listObjectVersionsRes = data[8]; - const { DeleteMarkers, Versions } = listObjectVersionsRes; + const listObjectVersionsRes = data[8]; + const { DeleteMarkers, Versions } = listObjectVersionsRes; - assert.strictEqual(DeleteMarkers, undefined); - assert.strictEqual(Versions.length, 2); + assert.strictEqual(DeleteMarkers, undefined); + assert.strictEqual(Versions.length, 2); - const currentVersion = Versions.find(v => v.IsLatest); - assertVersionIsNullAndUpdated(currentVersion); + const currentVersion = Versions.find(v => v.IsLatest); + assertVersionIsNullAndUpdated(currentVersion); - const nonCurrentVersion = Versions.find(v => !v.IsLatest); - assertVersionHasNotBeenUpdated(nonCurrentVersion, expectedVersionId); + const nonCurrentVersion = Versions.find(v => !v.IsLatest); + assertVersionHasNotBeenUpdated(nonCurrentVersion, expectedVersionId); - // give some time for the async deletes to complete - return setTimeout(() => checkVersionData(s3, bucket, keyName, expectedVersionId, testData, done), - 1000); - }); + // give some time for the async deletes to complete + return setTimeout( + () => checkVersionData(s3, bucket, keyName, expectedVersionId, testData, done), + 1000, + ); + }, + ); }); it('should update null version with no version id and versioning suspended', done => { let objMD; - return async.series([ - next => s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: keyName, - Body: Buffer.from(testData), - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Suspended' }, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => makeBackbeatRequest({ - method: 'GET', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', - }, - authCredentials: backbeatAuthCredentials, - }, (err, data) => { + return async.series( + [ + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: keyName, + Body: Buffer.from(testData), + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Suspended' }, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + makeBackbeatRequest( + { + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + }, + (err, data) => { + if (err) { + return next(err); + } + const { error, result } = updateStorageClass(data, storageClass); + if (error) { + return next(error); + } + objMD = result; + return next(); + }, + ), + next => + makeBackbeatRequest( + { + method: 'PUT', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + requestBody: objMD, + }, + next, + ), + next => + s3 + .send( + new HeadObjectCommand({ + Bucket: bucket, + Key: keyName, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new ListObjectVersionsCommand({ + Bucket: bucket, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + ], + (err, data) => { if (err) { - return next(err); + return done(err); } - const { error, result } = updateStorageClass(data, storageClass); - if (error) { - return next(error); - } - objMD = result; - return next(); - }), - next => makeBackbeatRequest({ - method: 'PUT', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', - }, - authCredentials: backbeatAuthCredentials, - requestBody: objMD, - }, next), - next => s3.send(new HeadObjectCommand({ - Bucket: bucket, - Key: keyName, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new ListObjectVersionsCommand({ - Bucket: bucket, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - ], (err, data) => { - if (err) { - return done(err); - } - const headObjectRes = data[4]; - assert.strictEqual(headObjectRes.VersionId, 'null'); - assert.strictEqual(headObjectRes.StorageClass, storageClass); + const headObjectRes = data[4]; + assert.strictEqual(headObjectRes.VersionId, 'null'); + assert.strictEqual(headObjectRes.StorageClass, storageClass); - const listObjectVersionsRes = data[5]; - const { DeleteMarkers, Versions } = listObjectVersionsRes; - assert.strictEqual(DeleteMarkers, undefined); - assert.strictEqual(Versions.length, 1); + const listObjectVersionsRes = data[5]; + const { DeleteMarkers, Versions } = listObjectVersionsRes; + assert.strictEqual(DeleteMarkers, undefined); + assert.strictEqual(Versions.length, 1); - const currentVersion = Versions[0]; - assert(currentVersion.IsLatest); - assertVersionIsNullAndUpdated(currentVersion); + const currentVersion = Versions[0]; + assert(currentVersion.IsLatest); + assertVersionIsNullAndUpdated(currentVersion); - return done(); - }); + return done(); + }, + ); }); it('should update null version if versioning suspended and null version has a version id', done => { let objMD; - return async.series([ - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Suspended' }, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: keyName, - Body: Buffer.from(testData), - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => makeBackbeatRequest({ - method: 'GET', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', - }, - authCredentials: backbeatAuthCredentials, - }, (err, data) => { + return async.series( + [ + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Suspended' }, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: keyName, + Body: Buffer.from(testData), + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + makeBackbeatRequest( + { + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + }, + (err, data) => { + if (err) { + return next(err); + } + const { error, result } = updateStorageClass(data, storageClass); + if (error) { + return next(error); + } + objMD = result; + return next(); + }, + ), + next => + makeBackbeatRequest( + { + method: 'PUT', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + requestBody: objMD, + }, + next, + ), + next => + s3 + .send( + new HeadObjectCommand({ + Bucket: bucket, + Key: keyName, + VersionId: 'null', + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new ListObjectVersionsCommand({ + Bucket: bucket, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + ], + (err, data) => { if (err) { - return next(err); + return done(err); } - const { error, result } = updateStorageClass(data, storageClass); - if (error) { - return next(error); - } - objMD = result; - return next(); - }), - next => makeBackbeatRequest({ - method: 'PUT', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', - }, - authCredentials: backbeatAuthCredentials, - requestBody: objMD, - }, next), - next => s3.send(new HeadObjectCommand({ - Bucket: bucket, - Key: keyName, - VersionId: 'null', - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new ListObjectVersionsCommand({ - Bucket: bucket, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - ], (err, data) => { - if (err) { - return done(err); - } - const headObjectRes = data[4]; - assert.strictEqual(headObjectRes.VersionId, 'null'); - assert.strictEqual(headObjectRes.StorageClass, storageClass); + const headObjectRes = data[4]; + assert.strictEqual(headObjectRes.VersionId, 'null'); + assert.strictEqual(headObjectRes.StorageClass, storageClass); - const listObjectVersionsRes = data[5]; - const { DeleteMarkers, Versions } = listObjectVersionsRes; - assert.strictEqual(Versions.length, 1); - assert.strictEqual(DeleteMarkers, undefined); + const listObjectVersionsRes = data[5]; + const { DeleteMarkers, Versions } = listObjectVersionsRes; + assert.strictEqual(Versions.length, 1); + assert.strictEqual(DeleteMarkers, undefined); - const currentVersion = Versions[0]; - assert(currentVersion.IsLatest); - assertVersionIsNullAndUpdated(currentVersion); - return done(); - }); + const currentVersion = Versions[0]; + assert(currentVersion.IsLatest); + assertVersionIsNullAndUpdated(currentVersion); + return done(); + }, + ); }); - it('should update null version if versioning suspended and null version has a version id and' + - 'put object afterward', done => { - let objMD; - return async.series([ - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Suspended' }, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: keyName, - Body: Buffer.from(testData), - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => makeBackbeatRequest({ - method: 'GET', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', - }, - authCredentials: backbeatAuthCredentials, - }, (err, data) => { - if (err) { - return next(err); - } - const { error, result } = updateStorageClass(data, storageClass); - if (error) { - return next(error); - } - objMD = result; - return next(); - }), - next => makeBackbeatRequest({ - method: 'PUT', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', - }, - authCredentials: backbeatAuthCredentials, - requestBody: objMD, - }, next), - next => s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: keyName, - Body: Buffer.from(testData), - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new HeadObjectCommand({ - Bucket: bucket, - Key: keyName, - VersionId: 'null', - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new ListObjectVersionsCommand({ - Bucket: bucket, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new HeadObjectCommand({ - Bucket: bucket, - Key: keyName, - VersionId: 'null', - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - ], (err, data) => { - if (err) { - return done(err); - } - - const headObjectRes = data[5]; - assert.strictEqual(headObjectRes.VersionId, 'null'); - assert(!headObjectRes.StorageClass); + it( + 'should update null version if versioning suspended and null version has a version id and' + + 'put object afterward', + done => { + let objMD; + return async.series( + [ + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Suspended' }, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: keyName, + Body: Buffer.from(testData), + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + makeBackbeatRequest( + { + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + }, + (err, data) => { + if (err) { + return next(err); + } + const { error, result } = updateStorageClass(data, storageClass); + if (error) { + return next(error); + } + objMD = result; + return next(); + }, + ), + next => + makeBackbeatRequest( + { + method: 'PUT', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + requestBody: objMD, + }, + next, + ), + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: keyName, + Body: Buffer.from(testData), + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new HeadObjectCommand({ + Bucket: bucket, + Key: keyName, + VersionId: 'null', + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new ListObjectVersionsCommand({ + Bucket: bucket, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new HeadObjectCommand({ + Bucket: bucket, + Key: keyName, + VersionId: 'null', + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + ], + (err, data) => { + if (err) { + return done(err); + } - const listObjectVersionsRes = data[6]; - const { DeleteMarkers, Versions } = listObjectVersionsRes; - assert.strictEqual(DeleteMarkers, undefined); - assert.strictEqual(Versions.length, 1); + const headObjectRes = data[5]; + assert.strictEqual(headObjectRes.VersionId, 'null'); + assert(!headObjectRes.StorageClass); - const currentVersion = Versions[0]; - assert(currentVersion.IsLatest); - assertVersionHasNotBeenUpdated(currentVersion, 'null'); - return done(); - }); - }); + const listObjectVersionsRes = data[6]; + const { DeleteMarkers, Versions } = listObjectVersionsRes; + assert.strictEqual(DeleteMarkers, undefined); + assert.strictEqual(Versions.length, 1); - it('should update null version if versioning suspended and null version has a version id and' + - 'put version afterward', done => { - let objMD; - let expectedVersionId; - return async.series([ - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Suspended' }, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: keyName, - Body: Buffer.from(testData), - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => makeBackbeatRequest({ - method: 'GET', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', - }, - authCredentials: backbeatAuthCredentials, - }, (err, data) => { - if (err) { - return next(err); - } - const { error, result } = updateStorageClass(data, storageClass); - if (error) { - return next(error); - } - objMD = result; - return next(); - }), - next => makeBackbeatRequest({ - method: 'PUT', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', + const currentVersion = Versions[0]; + assert(currentVersion.IsLatest); + assertVersionHasNotBeenUpdated(currentVersion, 'null'); + return done(); }, - authCredentials: backbeatAuthCredentials, - requestBody: objMD, - }, next), - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Enabled' }, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: keyName, - Body: Buffer.from(testData), - })).then(data => { - expectedVersionId = data.VersionId; - return next(null, data); - }).catch(err => { - next(err); - }), - next => s3.send(new HeadObjectCommand({ - Bucket: bucket, - Key: keyName, - VersionId: 'null', - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new ListObjectVersionsCommand({ - Bucket: bucket, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - ], (err, data) => { - if (err) { - return done(err); - } + ); + }, + ); + + it( + 'should update null version if versioning suspended and null version has a version id and' + + 'put version afterward', + done => { + let objMD; + let expectedVersionId; + return async.series( + [ + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Suspended' }, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: keyName, + Body: Buffer.from(testData), + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + makeBackbeatRequest( + { + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + }, + (err, data) => { + if (err) { + return next(err); + } + const { error, result } = updateStorageClass(data, storageClass); + if (error) { + return next(error); + } + objMD = result; + return next(); + }, + ), + next => + makeBackbeatRequest( + { + method: 'PUT', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + requestBody: objMD, + }, + next, + ), + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: keyName, + Body: Buffer.from(testData), + }), + ) + .then(data => { + expectedVersionId = data.VersionId; + return next(null, data); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new HeadObjectCommand({ + Bucket: bucket, + Key: keyName, + VersionId: 'null', + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new ListObjectVersionsCommand({ + Bucket: bucket, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + ], + (err, data) => { + if (err) { + return done(err); + } - const headObjectRes = data[6]; - assert.strictEqual(headObjectRes.VersionId, 'null'); - assert.strictEqual(headObjectRes.StorageClass, storageClass); + const headObjectRes = data[6]; + assert.strictEqual(headObjectRes.VersionId, 'null'); + assert.strictEqual(headObjectRes.StorageClass, storageClass); - const listObjectVersionsRes = data[7]; - const { Versions } = listObjectVersionsRes; - assert.strictEqual(Versions.length, 2); + const listObjectVersionsRes = data[7]; + const { Versions } = listObjectVersionsRes; + assert.strictEqual(Versions.length, 2); - const [currentVersion] = Versions.filter(v => v.IsLatest); - assertVersionHasNotBeenUpdated(currentVersion, expectedVersionId); + const [currentVersion] = Versions.filter(v => v.IsLatest); + assertVersionHasNotBeenUpdated(currentVersion, expectedVersionId); - const [nonCurrentVersion] = Versions.filter(v => !v.IsLatest); - assertVersionIsNullAndUpdated(nonCurrentVersion); - return done(); - }); - }); + const [nonCurrentVersion] = Versions.filter(v => !v.IsLatest); + assertVersionIsNullAndUpdated(nonCurrentVersion); + return done(); + }, + ); + }, + ); it('should update non-current null version if versioning suspended', done => { let expectedVersionId; let objMD; - return async.series([ - next => s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: keyName, - Body: Buffer.from(testData), - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Enabled' }, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: keyName, - Body: Buffer.from(testData), - })).then(data => { - expectedVersionId = data.VersionId; - return next(null, data); - }).catch(err => { - next(err); - }), - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Suspended' }, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => makeBackbeatRequest({ - method: 'GET', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', - }, - authCredentials: backbeatAuthCredentials, - }, (err, data) => { + return async.series( + [ + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: keyName, + Body: Buffer.from(testData), + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: keyName, + Body: Buffer.from(testData), + }), + ) + .then(data => { + expectedVersionId = data.VersionId; + return next(null, data); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Suspended' }, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + makeBackbeatRequest( + { + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + }, + (err, data) => { + if (err) { + return next(err); + } + const { error, result } = updateStorageClass(data, storageClass); + if (error) { + return next(error); + } + objMD = result; + return next(); + }, + ), + next => + makeBackbeatRequest( + { + method: 'PUT', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + requestBody: objMD, + }, + next, + ), + next => + s3 + .send( + new HeadObjectCommand({ + Bucket: bucket, + Key: keyName, + VersionId: 'null', + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new ListObjectVersionsCommand({ + Bucket: bucket, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new HeadObjectCommand({ + Bucket: bucket, + Key: keyName, + VersionId: 'null', + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + ], + (err, data) => { if (err) { - return next(err); + return done(err); } - const { error, result } = updateStorageClass(data, storageClass); - if (error) { - return next(error); - } - objMD = result; - return next(); - }), - next => makeBackbeatRequest({ - method: 'PUT', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', - }, - authCredentials: backbeatAuthCredentials, - requestBody: objMD, - }, next), - next => s3.send(new HeadObjectCommand({ - Bucket: bucket, - Key: keyName, - VersionId: 'null', - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new ListObjectVersionsCommand({ - Bucket: bucket, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new HeadObjectCommand({ - Bucket: bucket, - Key: keyName, - VersionId: 'null', - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - ], (err, data) => { - if (err) { - return done(err); - } - const headObjectRes = data[6]; - assert.strictEqual(headObjectRes.VersionId, 'null'); - assert.strictEqual(headObjectRes.StorageClass, storageClass); + const headObjectRes = data[6]; + assert.strictEqual(headObjectRes.VersionId, 'null'); + assert.strictEqual(headObjectRes.StorageClass, storageClass); - const listObjectVersionsRes = data[7]; - const deleteMarkers = listObjectVersionsRes.DeleteMarkers; - assert.strictEqual(deleteMarkers, undefined); - const { Versions } = listObjectVersionsRes; - assert.strictEqual(Versions.length, 2); + const listObjectVersionsRes = data[7]; + const deleteMarkers = listObjectVersionsRes.DeleteMarkers; + assert.strictEqual(deleteMarkers, undefined); + const { Versions } = listObjectVersionsRes; + assert.strictEqual(Versions.length, 2); - const [currentVersion] = Versions.filter(v => v.IsLatest); - assertVersionHasNotBeenUpdated(currentVersion, expectedVersionId); + const [currentVersion] = Versions.filter(v => v.IsLatest); + assertVersionHasNotBeenUpdated(currentVersion, expectedVersionId); - const [nonCurrentVersion] = Versions.filter(v => !v.IsLatest); - assertVersionIsNullAndUpdated(nonCurrentVersion); + const [nonCurrentVersion] = Versions.filter(v => !v.IsLatest); + assertVersionIsNullAndUpdated(nonCurrentVersion); - return done(); - }); + return done(); + }, + ); }); it('should update current null version if versioning suspended', done => { let objMD; let expectedVersionId; - return async.series([ - next => s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: keyName, - Body: Buffer.from(testData), - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Enabled' }, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: keyName, - Body: Buffer.from(testData), - })).then(result => { - expectedVersionId = result.VersionId; - return next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Suspended' }, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: keyName, - VersionId: expectedVersionId, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => makeBackbeatRequest({ - method: 'GET', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', - }, - authCredentials: backbeatAuthCredentials, - }, (err, data) => { + return async.series( + [ + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: keyName, + Body: Buffer.from(testData), + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: keyName, + Body: Buffer.from(testData), + }), + ) + .then(result => { + expectedVersionId = result.VersionId; + return next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Suspended' }, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: keyName, + VersionId: expectedVersionId, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + makeBackbeatRequest( + { + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + }, + (err, data) => { + if (err) { + return next(err); + } + const { error, result } = updateStorageClass(data, storageClass); + if (error) { + return next(error); + } + objMD = result; + return next(); + }, + ), + next => + makeBackbeatRequest( + { + method: 'PUT', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + requestBody: objMD, + }, + next, + ), + next => + s3 + .send( + new HeadObjectCommand({ + Bucket: bucket, + Key: keyName, + VersionId: 'null', + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new ListObjectVersionsCommand({ + Bucket: bucket, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + ], + (err, data) => { if (err) { - return next(err); - } - const { error, result } = updateStorageClass(data, storageClass); - if (error) { - return next(error); + return done(err); } - objMD = result; - return next(); - }), - next => makeBackbeatRequest({ - method: 'PUT', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', - }, - authCredentials: backbeatAuthCredentials, - requestBody: objMD, - }, next), - next => s3.send(new HeadObjectCommand({ - Bucket: bucket, - Key: keyName, - VersionId: 'null', - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new ListObjectVersionsCommand({ - Bucket: bucket, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - ], (err, data) => { - if (err) { - return done(err); - } - const headObjectRes = data[7]; - assert.strictEqual(headObjectRes.VersionId, 'null'); - assert.strictEqual(headObjectRes.StorageClass, storageClass); - - const listObjectVersionsRes = data[8]; - const { DeleteMarkers, Versions } = listObjectVersionsRes; - assert.strictEqual(Versions.length, 1); - assert.strictEqual(DeleteMarkers, undefined); - - const currentVersion = Versions[0]; - assert(currentVersion.IsLatest); - assertVersionIsNullAndUpdated(currentVersion); - return done(); - }); + const headObjectRes = data[7]; + assert.strictEqual(headObjectRes.VersionId, 'null'); + assert.strictEqual(headObjectRes.StorageClass, storageClass); + + const listObjectVersionsRes = data[8]; + const { DeleteMarkers, Versions } = listObjectVersionsRes; + assert.strictEqual(Versions.length, 1); + assert.strictEqual(DeleteMarkers, undefined); + + const currentVersion = Versions[0]; + assert(currentVersion.IsLatest); + assertVersionIsNullAndUpdated(currentVersion); + return done(); + }, + ); }); - it('should update current null version if versioning suspended and put a null version ' + - 'afterwards', done => { - let objMD; - let deletedVersionId; - return async.series([ - next => s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: keyName, - Body: Buffer.from(testData), - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Enabled' }, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: keyName, - Body: Buffer.from(testData), - })).then(data => { - deletedVersionId = data.VersionId; - return next(null, data); - }).catch(err => { - next(err); - }), - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Suspended' }, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: keyName, - VersionId: deletedVersionId, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => makeBackbeatRequest({ - method: 'GET', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', - }, - authCredentials: backbeatAuthCredentials, - }, (err, data) => { - if (err) { - return next(err); - } - const { error, result } = updateStorageClass(data, storageClass); - if (error) { - return next(error); - } - objMD = result; - return next(); - }), - next => makeBackbeatRequest({ - method: 'PUT', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', - }, - authCredentials: backbeatAuthCredentials, - requestBody: objMD, - }, next), - next => s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: keyName, - Body: Buffer.from(testData), - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new HeadObjectCommand({ - Bucket: bucket, - Key: keyName, - VersionId: 'null', - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new ListObjectVersionsCommand({ - Bucket: bucket, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new HeadObjectCommand({ - Bucket: bucket, - Key: keyName, - VersionId: 'null', - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - ], (err, data) => { - if (err) { - return done(err); - } + it( + 'should update current null version if versioning suspended and put a null version ' + 'afterwards', + done => { + let objMD; + let deletedVersionId; + return async.series( + [ + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: keyName, + Body: Buffer.from(testData), + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: keyName, + Body: Buffer.from(testData), + }), + ) + .then(data => { + deletedVersionId = data.VersionId; + return next(null, data); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Suspended' }, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: keyName, + VersionId: deletedVersionId, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + makeBackbeatRequest( + { + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + }, + (err, data) => { + if (err) { + return next(err); + } + const { error, result } = updateStorageClass(data, storageClass); + if (error) { + return next(error); + } + objMD = result; + return next(); + }, + ), + next => + makeBackbeatRequest( + { + method: 'PUT', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + requestBody: objMD, + }, + next, + ), + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: keyName, + Body: Buffer.from(testData), + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new HeadObjectCommand({ + Bucket: bucket, + Key: keyName, + VersionId: 'null', + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new ListObjectVersionsCommand({ + Bucket: bucket, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new HeadObjectCommand({ + Bucket: bucket, + Key: keyName, + VersionId: 'null', + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + ], + (err, data) => { + if (err) { + return done(err); + } - const headObjectRes = data[8]; - assert.strictEqual(headObjectRes.VersionId, 'null'); - assert(!headObjectRes.StorageClass); + const headObjectRes = data[8]; + assert.strictEqual(headObjectRes.VersionId, 'null'); + assert(!headObjectRes.StorageClass); - const listObjectVersionsRes = data[9]; - const { DeleteMarkers, Versions } = listObjectVersionsRes; - assert.strictEqual(DeleteMarkers, undefined); - assert.strictEqual(Versions.length, 1); + const listObjectVersionsRes = data[9]; + const { DeleteMarkers, Versions } = listObjectVersionsRes; + assert.strictEqual(DeleteMarkers, undefined); + assert.strictEqual(Versions.length, 1); - const currentVersion = Versions[0]; - assert(currentVersion.IsLatest); - assertVersionHasNotBeenUpdated(currentVersion, 'null'); + const currentVersion = Versions[0]; + assert(currentVersion.IsLatest); + assertVersionHasNotBeenUpdated(currentVersion, 'null'); - return done(); - }); - }); + return done(); + }, + ); + }, + ); it('should update current null version if versioning suspended and put a version afterwards', done => { let objMD; let deletedVersionId; let expectedVersionId; - return async.series([ - next => s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: keyName, - Body: Buffer.from(testData), - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Enabled' }, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: keyName, - Body: Buffer.from(testData), - })).then(result => { - deletedVersionId = result.VersionId; - return next(); - }).catch(err => { - next(err); - }), - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Suspended' }, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: keyName, - VersionId: deletedVersionId, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => makeBackbeatRequest({ - method: 'GET', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', - }, - authCredentials: backbeatAuthCredentials, - }, (err, data) => { + return async.series( + [ + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: keyName, + Body: Buffer.from(testData), + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: keyName, + Body: Buffer.from(testData), + }), + ) + .then(result => { + deletedVersionId = result.VersionId; + return next(); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Suspended' }, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: keyName, + VersionId: deletedVersionId, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + makeBackbeatRequest( + { + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + }, + (err, data) => { + if (err) { + return next(err); + } + const { error, result } = updateStorageClass(data, storageClass); + if (error) { + return next(error); + } + objMD = result; + return next(); + }, + ), + next => + makeBackbeatRequest( + { + method: 'PUT', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + requestBody: objMD, + }, + next, + ), + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: keyName, + Body: Buffer.from(testData), + }), + ) + .then(result => { + expectedVersionId = result.VersionId; + return next(); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new HeadObjectCommand({ + Bucket: bucket, + Key: keyName, + VersionId: 'null', + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new ListObjectVersionsCommand({ + Bucket: bucket, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }), + ], + (err, data) => { if (err) { - return next(err); - } - const { error, result } = updateStorageClass(data, storageClass); - if (error) { - return next(error); + return done(err); } - objMD = result; - return next(); - }), - next => makeBackbeatRequest({ - method: 'PUT', - resourceType: 'metadata', - bucket, - objectKey: keyName, - queryObj: { - versionId: 'null', - }, - authCredentials: backbeatAuthCredentials, - requestBody: objMD, - }, next), - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Enabled' }, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: keyName, - Body: Buffer.from(testData), - })).then(result => { - expectedVersionId = result.VersionId; - return next(); - }).catch(err => { - next(err); - }), - next => s3.send(new HeadObjectCommand({ - Bucket: bucket, - Key: keyName, - VersionId: 'null', - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - next => s3.send(new ListObjectVersionsCommand({ - Bucket: bucket, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }), - ], (err, data) => { - if (err) { - return done(err); - } - const headObjectRes = data[9]; - assert.strictEqual(headObjectRes.VersionId, 'null'); - assert.strictEqual(headObjectRes.StorageClass, storageClass); + const headObjectRes = data[9]; + assert.strictEqual(headObjectRes.VersionId, 'null'); + assert.strictEqual(headObjectRes.StorageClass, storageClass); - const listObjectVersionsRes = data[10]; - const { DeleteMarkers, Versions } = listObjectVersionsRes; - assert.strictEqual(DeleteMarkers, undefined); - assert.strictEqual(Versions.length, 2); + const listObjectVersionsRes = data[10]; + const { DeleteMarkers, Versions } = listObjectVersionsRes; + assert.strictEqual(DeleteMarkers, undefined); + assert.strictEqual(Versions.length, 2); - const [currentVersion] = Versions.filter(v => v.IsLatest); - assertVersionHasNotBeenUpdated(currentVersion, expectedVersionId); + const [currentVersion] = Versions.filter(v => v.IsLatest); + assertVersionHasNotBeenUpdated(currentVersion, expectedVersionId); - const [nonCurrentVersion] = Versions.filter(v => !v.IsLatest); - assertVersionIsNullAndUpdated(nonCurrentVersion); + const [nonCurrentVersion] = Versions.filter(v => !v.IsLatest); + assertVersionIsNullAndUpdated(nonCurrentVersion); - return done(); - }); + return done(); + }, + ); }); }); describe('backbeat PUT routes', () => { - describe('PUT data + metadata should create a new complete object', - () => { - [{ - caption: 'with ascii test key', - key: testKey, encodedKey: testKey, - }, - { - caption: 'with UTF8 key', - key: testKeyUTF8, encodedKey: encodeURI(testKeyUTF8), - }, - { - caption: 'with percents and spaces encoded as \'+\' in key', - key: '50% full or 50% empty', - encodedKey: '50%25+full+or+50%25+empty', - }, - { - caption: 'with legacy API v1', - key: testKey, encodedKey: testKey, - legacyAPI: true, - }, - { - caption: 'with encryption configuration', - key: testKey, encodedKey: testKey, - encryption: true, - }, - { - caption: 'with encryption configuration and legacy API v1', - key: testKey, encodedKey: testKey, - encryption: true, - legacyAPI: true, - }].concat([ - `${testKeyUTF8}/${testKeyUTF8}/%42/mykey`, - 'Pâtisserie=中文-español-English', - 'notes/spring/1.txt', - 'notes/spring/2.txt', - 'notes/spring/march/1.txt', - 'notes/summer/1.txt', - 'notes/summer/2.txt', - 'notes/summer/august/1.txt', - 'notes/year.txt', - 'notes/yore.rs', - 'notes/zaphod/Beeblebrox.txt', - ].map(key => ({ - key, encodedKey: encodeURI(key), - caption: `with key ${key}`, - }))) - .forEach(testCase => { - it(testCase.caption, done => { - async.waterfall([next => { - const queryObj = testCase.legacyAPI ? {} : { v2: '' }; - makeBackbeatRequest({ - method: 'PUT', bucket: testCase.encryption ? - TEST_ENCRYPTED_BUCKET : TEST_BUCKET, - objectKey: testCase.encodedKey, - resourceType: 'data', - queryObj, - headers: { - 'content-length': testData.length, - 'x-scal-canonical-id': testArn, + describe('PUT data + metadata should create a new complete object', () => { + [ + { + caption: 'with ascii test key', + key: testKey, + encodedKey: testKey, + }, + { + caption: 'with UTF8 key', + key: testKeyUTF8, + encodedKey: encodeURI(testKeyUTF8), + }, + { + caption: "with percents and spaces encoded as '+' in key", + key: '50% full or 50% empty', + encodedKey: '50%25+full+or+50%25+empty', + }, + { + caption: 'with legacy API v1', + key: testKey, + encodedKey: testKey, + legacyAPI: true, + }, + { + caption: 'with encryption configuration', + key: testKey, + encodedKey: testKey, + encryption: true, + }, + { + caption: 'with encryption configuration and legacy API v1', + key: testKey, + encodedKey: testKey, + encryption: true, + legacyAPI: true, + }, + ] + .concat( + [ + `${testKeyUTF8}/${testKeyUTF8}/%42/mykey`, + 'Pâtisserie=中文-español-English', + 'notes/spring/1.txt', + 'notes/spring/2.txt', + 'notes/spring/march/1.txt', + 'notes/summer/1.txt', + 'notes/summer/2.txt', + 'notes/summer/august/1.txt', + 'notes/year.txt', + 'notes/yore.rs', + 'notes/zaphod/Beeblebrox.txt', + ].map(key => ({ + key, + encodedKey: encodeURI(key), + caption: `with key ${key}`, + })), + ) + .forEach(testCase => { + it(testCase.caption, done => { + async.waterfall( + [ + next => { + const queryObj = testCase.legacyAPI ? {} : { v2: '' }; + makeBackbeatRequest( + { + method: 'PUT', + bucket: testCase.encryption ? TEST_ENCRYPTED_BUCKET : TEST_BUCKET, + objectKey: testCase.encodedKey, + resourceType: 'data', + queryObj, + headers: { + 'content-length': testData.length, + 'x-scal-canonical-id': testArn, + }, + authCredentials: backbeatAuthCredentials, + requestBody: testData, + }, + next, + ); + }, + (response, next) => { + assert.strictEqual(response.statusCode, 200); + const newMd = getMetadataToPut(response); + if (testCase.encryption && !testCase.legacyAPI) { + assert.strictEqual(typeof newMd.location[0].cryptoScheme, 'number'); + assert.strictEqual(typeof newMd.location[0].cipheredDataKey, 'string'); + } else { + // if no encryption or legacy API, data should not be encrypted + assert.strictEqual(newMd.location[0].cryptoScheme, undefined); + assert.strictEqual(newMd.location[0].cipheredDataKey, undefined); + } + makeBackbeatRequest( + { + method: 'PUT', + bucket: testCase.encryption ? TEST_ENCRYPTED_BUCKET : TEST_BUCKET, + objectKey: testCase.encodedKey, + resourceType: 'metadata', + authCredentials: backbeatAuthCredentials, + requestBody: JSON.stringify(newMd), + }, + next, + ); + }, + (response, next) => { + assert.strictEqual(response.statusCode, 200); + checkObjectData( + s3, + testCase.encryption ? TEST_ENCRYPTED_BUCKET : TEST_BUCKET, + testCase.key, + testData, + next, + ); + }, + ], + err => { + assert.ifError(err); + done(); }, - authCredentials: backbeatAuthCredentials, - requestBody: testData }, next); - }, (response, next) => { - assert.strictEqual(response.statusCode, 200); - const newMd = getMetadataToPut(response); - if (testCase.encryption && !testCase.legacyAPI) { - assert.strictEqual(typeof newMd.location[0].cryptoScheme, 'number'); - assert.strictEqual(typeof newMd.location[0].cipheredDataKey, 'string'); - } else { - // if no encryption or legacy API, data should not be encrypted - assert.strictEqual(newMd.location[0].cryptoScheme, undefined); - assert.strictEqual(newMd.location[0].cipheredDataKey, undefined); - } - makeBackbeatRequest({ - method: 'PUT', bucket: testCase.encryption ? - TEST_ENCRYPTED_BUCKET : TEST_BUCKET, - objectKey: testCase.encodedKey, - resourceType: 'metadata', - authCredentials: backbeatAuthCredentials, - requestBody: JSON.stringify(newMd), - }, next); - }, (response, next) => { - assert.strictEqual(response.statusCode, 200); - checkObjectData( - s3, testCase.encryption ? TEST_ENCRYPTED_BUCKET : TEST_BUCKET, - testCase.key, testData, next); - }], err => { - assert.ifError(err); - done(); + ); }); }); - }); }); it('should PUT metadata for a non-versioned bucket', done => { const bucket = NONVERSIONED_BUCKET; const objectKey = 'non-versioned-key'; - async.waterfall([ - next => - makeBackbeatRequest({ - method: 'PUT', - bucket, - objectKey, - resourceType: 'data', - queryObj: { v2: '' }, - headers: { - 'content-length': testData.length, - 'content-md5': testDataMd5, - 'x-scal-canonical-id': testArn, - }, - authCredentials: backbeatAuthCredentials, - requestBody: testData, - }, (err, response) => { - assert.ifError(err); - const metadata = Object.assign({}, nonVersionedTestMd, { - location: JSON.parse(response.body), - }); - return next(null, metadata); - }), - (metadata, next) => - makeBackbeatRequest({ - method: 'PUT', - bucket, - objectKey, - resourceType: 'metadata', - authCredentials: backbeatAuthCredentials, - requestBody: JSON.stringify(metadata), - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - next(); - }), - next => - s3.send(new HeadObjectCommand({ - Bucket: bucket, - Key: objectKey, - })).then(result => { - assert.strictEqual(result.StorageClass, 'awsbackend'); - next(); - }).catch(err => { - next(err); - }), - next => checkObjectData(s3, bucket, objectKey, testData, next), - ], done); + async.waterfall( + [ + next => + makeBackbeatRequest( + { + method: 'PUT', + bucket, + objectKey, + resourceType: 'data', + queryObj: { v2: '' }, + headers: { + 'content-length': testData.length, + 'content-md5': testDataMd5, + 'x-scal-canonical-id': testArn, + }, + authCredentials: backbeatAuthCredentials, + requestBody: testData, + }, + (err, response) => { + assert.ifError(err); + const metadata = Object.assign({}, nonVersionedTestMd, { + location: JSON.parse(response.body), + }); + return next(null, metadata); + }, + ), + (metadata, next) => + makeBackbeatRequest( + { + method: 'PUT', + bucket, + objectKey, + resourceType: 'metadata', + authCredentials: backbeatAuthCredentials, + requestBody: JSON.stringify(metadata), + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + next(); + }, + ), + next => + s3 + .send( + new HeadObjectCommand({ + Bucket: bucket, + Key: objectKey, + }), + ) + .then(result => { + assert.strictEqual(result.StorageClass, 'awsbackend'); + next(); + }) + .catch(err => { + next(err); + }), + next => checkObjectData(s3, bucket, objectKey, testData, next), + ], + done, + ); }); - it('PUT metadata with "x-scal-replication-content: METADATA"' + - 'header should replicate metadata only', done => { - async.waterfall([next => { - makeBackbeatRequest({ - method: 'PUT', bucket: TEST_ENCRYPTED_BUCKET, - objectKey: 'test-updatemd-key', - resourceType: 'data', - queryObj: { v2: '' }, - headers: { - 'content-length': testData.length, - 'x-scal-canonical-id': testArn, + it( + 'PUT metadata with "x-scal-replication-content: METADATA"' + 'header should replicate metadata only', + done => { + async.waterfall( + [ + next => { + makeBackbeatRequest( + { + method: 'PUT', + bucket: TEST_ENCRYPTED_BUCKET, + objectKey: 'test-updatemd-key', + resourceType: 'data', + queryObj: { v2: '' }, + headers: { + 'content-length': testData.length, + 'x-scal-canonical-id': testArn, + }, + authCredentials: backbeatAuthCredentials, + requestBody: testData, + }, + next, + ); + }, + (response, next) => { + assert.strictEqual(response.statusCode, 200); + const newMd = getMetadataToPut(response); + makeBackbeatRequest( + { + method: 'PUT', + bucket: TEST_ENCRYPTED_BUCKET, + objectKey: 'test-updatemd-key', + resourceType: 'metadata', + authCredentials: backbeatAuthCredentials, + requestBody: JSON.stringify(newMd), + }, + next, + ); + }, + (response, next) => { + assert.strictEqual(response.statusCode, 200); + // Don't update the sent metadata since it is sent by + // backbeat as received from the replication queue, + // without updated data location or encryption info + // (since that info is not known by backbeat) + const newMd = Object.assign({}, testMd); + makeBackbeatRequest( + { + method: 'PUT', + bucket: TEST_ENCRYPTED_BUCKET, + objectKey: 'test-updatemd-key', + resourceType: 'metadata', + headers: { 'x-scal-replication-content': 'METADATA' }, + authCredentials: backbeatAuthCredentials, + requestBody: JSON.stringify(newMd), + }, + next, + ); + }, + (response, next) => { + assert.strictEqual(response.statusCode, 200); + checkObjectData(s3, TEST_ENCRYPTED_BUCKET, 'test-updatemd-key', testData, next); + }, + ], + err => { + assert.ifError(err); + done(); }, - authCredentials: backbeatAuthCredentials, - requestBody: testData, - }, next); - }, (response, next) => { - assert.strictEqual(response.statusCode, 200); - const newMd = getMetadataToPut(response); - makeBackbeatRequest({ - method: 'PUT', bucket: TEST_ENCRYPTED_BUCKET, - objectKey: 'test-updatemd-key', - resourceType: 'metadata', - authCredentials: backbeatAuthCredentials, - requestBody: JSON.stringify(newMd), - }, next); - }, (response, next) => { - assert.strictEqual(response.statusCode, 200); - // Don't update the sent metadata since it is sent by - // backbeat as received from the replication queue, - // without updated data location or encryption info - // (since that info is not known by backbeat) - const newMd = Object.assign({}, testMd); - makeBackbeatRequest({ - method: 'PUT', bucket: TEST_ENCRYPTED_BUCKET, - objectKey: 'test-updatemd-key', - resourceType: 'metadata', - headers: { 'x-scal-replication-content': 'METADATA' }, - authCredentials: backbeatAuthCredentials, - requestBody: JSON.stringify(newMd), - }, next); - }, (response, next) => { - assert.strictEqual(response.statusCode, 200); - checkObjectData(s3, TEST_ENCRYPTED_BUCKET, 'test-updatemd-key', - testData, next); - }], err => { - assert.ifError(err); - done(); - }); - }); + ); + }, + ); itIfLocationAws('should PUT tags for a non-versioned bucket (awslocation)', function test(done) { this.timeout(10000); const bucket = NONVERSIONED_BUCKET; const awsKey = uuidv4(); - async.waterfall([ - next => - makeBackbeatRequest({ - method: 'PUT', - bucket, - objectKey: awsKey, - resourceType: 'multiplebackenddata', - queryObj: { operation: 'putobject' }, - headers: { - 'content-length': testData.length, - 'x-scal-canonical-id': testArn, - 'x-scal-storage-type': 'aws_s3', - 'x-scal-storage-class': awsLocation, - 'x-scal-tags': JSON.stringify({ Key1: 'Value1' }), - }, - authCredentials: backbeatAuthCredentials, - requestBody: testData, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - return next(); - }), - next => - awsClient.send(new GetObjectTaggingCommand({ - Bucket: awsBucket, - Key: awsKey, - })).then(data => { - assert.deepStrictEqual(data.TagSet, [{ - Key: 'Key1', - Value: 'Value1' - }]); - next(null, data); - }).catch(err => { - next(err); - }), - ], done); + async.waterfall( + [ + next => + makeBackbeatRequest( + { + method: 'PUT', + bucket, + objectKey: awsKey, + resourceType: 'multiplebackenddata', + queryObj: { operation: 'putobject' }, + headers: { + 'content-length': testData.length, + 'x-scal-canonical-id': testArn, + 'x-scal-storage-type': 'aws_s3', + 'x-scal-storage-class': awsLocation, + 'x-scal-tags': JSON.stringify({ Key1: 'Value1' }), + }, + authCredentials: backbeatAuthCredentials, + requestBody: testData, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + return next(); + }, + ), + next => + awsClient + .send( + new GetObjectTaggingCommand({ + Bucket: awsBucket, + Key: awsKey, + }), + ) + .then(data => { + assert.deepStrictEqual(data.TagSet, [ + { + Key: 'Key1', + Value: 'Value1', + }, + ]); + next(null, data); + }) + .catch(err => { + next(err); + }), + ], + done, + ); }); const testCases = [ @@ -2390,544 +3199,734 @@ describe('backbeat routes', () => { testCases.forEach(({ description, bucket }) => { it(`should PUT metadata and data if ${description} and x-scal-versioning-required is not set`, done => { let objectMd; - async.waterfall([ - next => s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: 'sourcekey', - Body: Buffer.from(testData), - })).then(res => next(null, res)).catch(err => next(err)), - (resp, next) => makeBackbeatRequest({ - method: 'GET', - resourceType: 'metadata', - bucket, - objectKey: 'sourcekey', - authCredentials: backbeatAuthCredentials, - }, (err, resp) => { - objectMd = JSON.parse(resp.body).Body; - return next(); - }), - next => { - makeBackbeatRequest({ - method: 'PUT', bucket, - objectKey: 'destinationkey', - resourceType: 'data', - queryObj: { v2: '' }, - headers: { - 'content-length': testData.length, - 'x-scal-canonical-id': testArn, - }, - authCredentials: backbeatAuthCredentials, - requestBody: testData, - }, next); - }, (response, next) => { - assert.strictEqual(response.statusCode, 200); - makeBackbeatRequest({ - method: 'PUT', bucket, - objectKey: 'destinationkey', - resourceType: 'metadata', - authCredentials: backbeatAuthCredentials, - requestBody: objectMd, - }, next); - }], + async.waterfall( + [ + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: 'sourcekey', + Body: Buffer.from(testData), + }), + ) + .then(res => next(null, res)) + .catch(err => next(err)), + (resp, next) => + makeBackbeatRequest( + { + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: 'sourcekey', + authCredentials: backbeatAuthCredentials, + }, + (err, resp) => { + objectMd = JSON.parse(resp.body).Body; + return next(); + }, + ), + next => { + makeBackbeatRequest( + { + method: 'PUT', + bucket, + objectKey: 'destinationkey', + resourceType: 'data', + queryObj: { v2: '' }, + headers: { + 'content-length': testData.length, + 'x-scal-canonical-id': testArn, + }, + authCredentials: backbeatAuthCredentials, + requestBody: testData, + }, + next, + ); + }, + (response, next) => { + assert.strictEqual(response.statusCode, 200); + makeBackbeatRequest( + { + method: 'PUT', + bucket, + objectKey: 'destinationkey', + resourceType: 'metadata', + authCredentials: backbeatAuthCredentials, + requestBody: objectMd, + }, + next, + ); + }, + ], err => { assert.ifError(err); done(); - }); + }, + ); }); }); testCases.forEach(({ description, bucket }) => { it(`should refuse PUT data if ${description} and x-scal-versioning-required is true`, done => { - makeBackbeatRequest({ - method: 'PUT', - bucket, - objectKey: testKey, - resourceType: 'data', - queryObj: { v2: '' }, - headers: { - 'content-length': testData.length, - 'x-scal-canonical-id': testArn, - 'x-scal-versioning-required': 'true', + makeBackbeatRequest( + { + method: 'PUT', + bucket, + objectKey: testKey, + resourceType: 'data', + queryObj: { v2: '' }, + headers: { + 'content-length': testData.length, + 'x-scal-canonical-id': testArn, + 'x-scal-versioning-required': 'true', + }, + authCredentials: backbeatAuthCredentials, + requestBody: testData, }, - authCredentials: backbeatAuthCredentials, - requestBody: testData, - }, err => { - assert.strictEqual(err.code, 'InvalidBucketState'); - done(); - }); + err => { + assert.strictEqual(err.code, 'InvalidBucketState'); + done(); + }, + ); }); }); testCases.forEach(({ description, bucket }) => { it(`should refuse PUT metadata if ${description} and x-scal-versioning-required is true`, done => { - makeBackbeatRequest({ - method: 'PUT', - bucket, - objectKey: testKey, - resourceType: 'metadata', - queryObj: { - versionId: versionIdUtils.encode(testMd.versionId), + makeBackbeatRequest( + { + method: 'PUT', + bucket, + objectKey: testKey, + resourceType: 'metadata', + queryObj: { + versionId: versionIdUtils.encode(testMd.versionId), + }, + headers: { + 'x-scal-versioning-required': 'true', + }, + authCredentials: backbeatAuthCredentials, + requestBody: JSON.stringify(testMd), }, - headers: { - 'x-scal-versioning-required': 'true', + err => { + assert.strictEqual(err.code, 'InvalidBucketState'); + done(); }, - authCredentials: backbeatAuthCredentials, - requestBody: JSON.stringify(testMd), - }, err => { - assert.strictEqual(err.code, 'InvalidBucketState'); - done(); - }); - }); - }); - - it('should refuse PUT data if no x-scal-canonical-id header ' + - 'is provided', done => makeBackbeatRequest({ - method: 'PUT', bucket: TEST_BUCKET, - objectKey: testKey, resourceType: 'data', - queryObj: { v2: '' }, - headers: { - 'content-length': testData.length, - }, - authCredentials: backbeatAuthCredentials, - requestBody: testData, - }, - err => { - assert.strictEqual(err.code, 'BadRequest'); - done(); - })); - - it('should refuse PUT in metadata-only mode if object does not exist', - done => { - async.waterfall([next => { - const newMd = Object.assign({}, testMd); - makeBackbeatRequest({ - method: 'PUT', bucket: TEST_BUCKET, - objectKey: 'does-not-exist', - resourceType: 'metadata', - headers: { 'x-scal-replication-content': 'METADATA' }, - authCredentials: backbeatAuthCredentials, - requestBody: JSON.stringify(newMd), - }, next); - }], err => { - assert.strictEqual(err.statusCode, 404); - done(); + ); }); }); - it('should remove old object data locations if version is overwritten ' + - 'with same contents', done => { - let oldLocation; - const testKeyOldData = `${testKey}-old-data`; - async.waterfall([next => { - // put object's data locations - makeBackbeatRequest({ - method: 'PUT', bucket: TEST_BUCKET, - objectKey: testKey, - resourceType: 'data', - headers: { - 'content-length': testData.length, - 'x-scal-canonical-id': testArn, - }, - authCredentials: backbeatAuthCredentials, - requestBody: testData }, next); - }, (response, next) => { - assert.strictEqual(response.statusCode, 200); - // put object metadata - const newMd = Object.assign({}, testMd); - newMd.location = JSON.parse(response.body); - oldLocation = newMd.location; - makeBackbeatRequest({ - method: 'PUT', bucket: TEST_BUCKET, - objectKey: testKey, - resourceType: 'metadata', - queryObj: { - versionId: versionIdUtils.encode(testMd.versionId), - }, - authCredentials: backbeatAuthCredentials, - requestBody: JSON.stringify(newMd), - }, next); - }, (response, next) => { - assert.strictEqual(response.statusCode, 200); - // put another object which metadata reference the - // same data locations, we will attempt to retrieve - // this object at the end of the test to confirm that - // its locations have been deleted - const oldDataMd = Object.assign({}, testMd); - oldDataMd.location = oldLocation; - makeBackbeatRequest({ - method: 'PUT', bucket: TEST_BUCKET, - objectKey: testKeyOldData, - resourceType: 'metadata', - authCredentials: backbeatAuthCredentials, - requestBody: JSON.stringify(oldDataMd), - }, next); - }, (response, next) => { - assert.strictEqual(response.statusCode, 200); - // create new data locations - makeBackbeatRequest({ - method: 'PUT', bucket: TEST_BUCKET, + it('should refuse PUT data if no x-scal-canonical-id header ' + 'is provided', done => + makeBackbeatRequest( + { + method: 'PUT', + bucket: TEST_BUCKET, objectKey: testKey, resourceType: 'data', + queryObj: { v2: '' }, headers: { 'content-length': testData.length, - 'x-scal-canonical-id': testArn, }, authCredentials: backbeatAuthCredentials, - requestBody: testData }, next); - }, (response, next) => { - assert.strictEqual(response.statusCode, 200); - // overwrite the original object version, now - // with references to the new data locations - const newMd = Object.assign({}, testMd); - newMd.location = JSON.parse(response.body); - makeBackbeatRequest({ - method: 'PUT', bucket: TEST_BUCKET, - objectKey: testKey, - resourceType: 'metadata', - queryObj: { - versionId: versionIdUtils.encode(testMd.versionId), + requestBody: testData, + }, + err => { + assert.strictEqual(err.code, 'BadRequest'); + done(); + }, + ), + ); + + it('should refuse PUT in metadata-only mode if object does not exist', done => { + async.waterfall( + [ + next => { + const newMd = Object.assign({}, testMd); + makeBackbeatRequest( + { + method: 'PUT', + bucket: TEST_BUCKET, + objectKey: 'does-not-exist', + resourceType: 'metadata', + headers: { 'x-scal-replication-content': 'METADATA' }, + authCredentials: backbeatAuthCredentials, + requestBody: JSON.stringify(newMd), + }, + next, + ); }, - authCredentials: backbeatAuthCredentials, - requestBody: JSON.stringify(newMd), - }, next); - }, (response, next) => { - assert.strictEqual(response.statusCode, 200); - // give some time for the async deletes to complete - setTimeout(() => checkObjectData(s3, TEST_BUCKET, testKey, testData, next), - 1000); - }, next => { - // check that the object copy referencing the old data - // locations is unreadable, confirming that the old - // data locations have been deleted - s3.send(new GetObjectCommand({ - Bucket: TEST_BUCKET, - Key: testKeyOldData, - })).catch(err => { - assert(err, 'expected error to get object with old data ' + - 'locations, got success'); - next(); - }); - }], err => { - assert.ifError(err); - done(); - }); + ], + err => { + assert.strictEqual(err.statusCode, 404); + done(); + }, + ); }); - it('should remove old object data locations if version is overwritten ' + - 'with empty contents', done => { + it('should remove old object data locations if version is overwritten ' + 'with same contents', done => { let oldLocation; const testKeyOldData = `${testKey}-old-data`; - async.waterfall([next => { - // put object's data locations - makeBackbeatRequest({ - method: 'PUT', bucket: TEST_BUCKET, - objectKey: testKey, - resourceType: 'data', - headers: { - 'content-length': testData.length, - 'x-scal-canonical-id': testArn, + async.waterfall( + [ + next => { + // put object's data locations + makeBackbeatRequest( + { + method: 'PUT', + bucket: TEST_BUCKET, + objectKey: testKey, + resourceType: 'data', + headers: { + 'content-length': testData.length, + 'x-scal-canonical-id': testArn, + }, + authCredentials: backbeatAuthCredentials, + requestBody: testData, + }, + next, + ); }, - authCredentials: backbeatAuthCredentials, - requestBody: testData }, next); - }, (response, next) => { - assert.strictEqual(response.statusCode, 200); - // put object metadata - const newMd = Object.assign({}, testMd); - newMd.location = JSON.parse(response.body); - oldLocation = newMd.location; - makeBackbeatRequest({ - method: 'PUT', bucket: TEST_BUCKET, - objectKey: testKey, - resourceType: 'metadata', - queryObj: { - versionId: versionIdUtils.encode(testMd.versionId), + (response, next) => { + assert.strictEqual(response.statusCode, 200); + // put object metadata + const newMd = Object.assign({}, testMd); + newMd.location = JSON.parse(response.body); + oldLocation = newMd.location; + makeBackbeatRequest( + { + method: 'PUT', + bucket: TEST_BUCKET, + objectKey: testKey, + resourceType: 'metadata', + queryObj: { + versionId: versionIdUtils.encode(testMd.versionId), + }, + authCredentials: backbeatAuthCredentials, + requestBody: JSON.stringify(newMd), + }, + next, + ); }, - authCredentials: backbeatAuthCredentials, - requestBody: JSON.stringify(newMd), - }, next); - }, (response, next) => { - assert.strictEqual(response.statusCode, 200); - // put another object which metadata reference the - // same data locations, we will attempt to retrieve - // this object at the end of the test to confirm that - // its locations have been deleted - const oldDataMd = Object.assign({}, testMd); - oldDataMd.location = oldLocation; - makeBackbeatRequest({ - method: 'PUT', bucket: TEST_BUCKET, - objectKey: testKeyOldData, - resourceType: 'metadata', - queryObj: { - versionId: versionIdUtils.encode(testMd.versionId), + (response, next) => { + assert.strictEqual(response.statusCode, 200); + // put another object which metadata reference the + // same data locations, we will attempt to retrieve + // this object at the end of the test to confirm that + // its locations have been deleted + const oldDataMd = Object.assign({}, testMd); + oldDataMd.location = oldLocation; + makeBackbeatRequest( + { + method: 'PUT', + bucket: TEST_BUCKET, + objectKey: testKeyOldData, + resourceType: 'metadata', + authCredentials: backbeatAuthCredentials, + requestBody: JSON.stringify(oldDataMd), + }, + next, + ); }, - authCredentials: backbeatAuthCredentials, - requestBody: JSON.stringify(oldDataMd), - }, next); - }, (response, next) => { - assert.strictEqual(response.statusCode, 200); - // overwrite the original object version with an empty location - const newMd = Object.assign({}, testMd); - newMd['content-length'] = 0; - newMd['content-md5'] = emptyContentsMd5; - newMd.location = null; - makeBackbeatRequest({ - method: 'PUT', bucket: TEST_BUCKET, - objectKey: testKey, - resourceType: 'metadata', - queryObj: { - versionId: versionIdUtils.encode(testMd.versionId), + (response, next) => { + assert.strictEqual(response.statusCode, 200); + // create new data locations + makeBackbeatRequest( + { + method: 'PUT', + bucket: TEST_BUCKET, + objectKey: testKey, + resourceType: 'data', + headers: { + 'content-length': testData.length, + 'x-scal-canonical-id': testArn, + }, + authCredentials: backbeatAuthCredentials, + requestBody: testData, + }, + next, + ); }, - authCredentials: backbeatAuthCredentials, - requestBody: JSON.stringify(newMd), - }, next); - }, (response, next) => { - assert.strictEqual(response.statusCode, 200); - // give some time for the async deletes to complete - setTimeout(() => checkObjectData(s3, TEST_BUCKET, testKey, '', next), - 1000); - }, next => { - // check that the object copy referencing the old data - // locations is unreadable, confirming that the old - // data locations have been deleted - s3.send(new GetObjectCommand({ - Bucket: TEST_BUCKET, - Key: testKeyOldData, - })).catch(err => { - assert(err, 'expected error to get object with old data ' + - 'locations, got success'); - next(); - }); - }], err => { - assert.ifError(err); - done(); - }); + (response, next) => { + assert.strictEqual(response.statusCode, 200); + // overwrite the original object version, now + // with references to the new data locations + const newMd = Object.assign({}, testMd); + newMd.location = JSON.parse(response.body); + makeBackbeatRequest( + { + method: 'PUT', + bucket: TEST_BUCKET, + objectKey: testKey, + resourceType: 'metadata', + queryObj: { + versionId: versionIdUtils.encode(testMd.versionId), + }, + authCredentials: backbeatAuthCredentials, + requestBody: JSON.stringify(newMd), + }, + next, + ); + }, + (response, next) => { + assert.strictEqual(response.statusCode, 200); + // give some time for the async deletes to complete + setTimeout(() => checkObjectData(s3, TEST_BUCKET, testKey, testData, next), 1000); + }, + next => { + // check that the object copy referencing the old data + // locations is unreadable, confirming that the old + // data locations have been deleted + s3.send( + new GetObjectCommand({ + Bucket: TEST_BUCKET, + Key: testKeyOldData, + }), + ).catch(err => { + assert(err, 'expected error to get object with old data ' + 'locations, got success'); + next(); + }); + }, + ], + err => { + assert.ifError(err); + done(); + }, + ); }); - it('should not remove data locations on replayed metadata PUT', - done => { + it('should remove old object data locations if version is overwritten ' + 'with empty contents', done => { + let oldLocation; + const testKeyOldData = `${testKey}-old-data`; + async.waterfall( + [ + next => { + // put object's data locations + makeBackbeatRequest( + { + method: 'PUT', + bucket: TEST_BUCKET, + objectKey: testKey, + resourceType: 'data', + headers: { + 'content-length': testData.length, + 'x-scal-canonical-id': testArn, + }, + authCredentials: backbeatAuthCredentials, + requestBody: testData, + }, + next, + ); + }, + (response, next) => { + assert.strictEqual(response.statusCode, 200); + // put object metadata + const newMd = Object.assign({}, testMd); + newMd.location = JSON.parse(response.body); + oldLocation = newMd.location; + makeBackbeatRequest( + { + method: 'PUT', + bucket: TEST_BUCKET, + objectKey: testKey, + resourceType: 'metadata', + queryObj: { + versionId: versionIdUtils.encode(testMd.versionId), + }, + authCredentials: backbeatAuthCredentials, + requestBody: JSON.stringify(newMd), + }, + next, + ); + }, + (response, next) => { + assert.strictEqual(response.statusCode, 200); + // put another object which metadata reference the + // same data locations, we will attempt to retrieve + // this object at the end of the test to confirm that + // its locations have been deleted + const oldDataMd = Object.assign({}, testMd); + oldDataMd.location = oldLocation; + makeBackbeatRequest( + { + method: 'PUT', + bucket: TEST_BUCKET, + objectKey: testKeyOldData, + resourceType: 'metadata', + queryObj: { + versionId: versionIdUtils.encode(testMd.versionId), + }, + authCredentials: backbeatAuthCredentials, + requestBody: JSON.stringify(oldDataMd), + }, + next, + ); + }, + (response, next) => { + assert.strictEqual(response.statusCode, 200); + // overwrite the original object version with an empty location + const newMd = Object.assign({}, testMd); + newMd['content-length'] = 0; + newMd['content-md5'] = emptyContentsMd5; + newMd.location = null; + makeBackbeatRequest( + { + method: 'PUT', + bucket: TEST_BUCKET, + objectKey: testKey, + resourceType: 'metadata', + queryObj: { + versionId: versionIdUtils.encode(testMd.versionId), + }, + authCredentials: backbeatAuthCredentials, + requestBody: JSON.stringify(newMd), + }, + next, + ); + }, + (response, next) => { + assert.strictEqual(response.statusCode, 200); + // give some time for the async deletes to complete + setTimeout(() => checkObjectData(s3, TEST_BUCKET, testKey, '', next), 1000); + }, + next => { + // check that the object copy referencing the old data + // locations is unreadable, confirming that the old + // data locations have been deleted + s3.send( + new GetObjectCommand({ + Bucket: TEST_BUCKET, + Key: testKeyOldData, + }), + ).catch(err => { + assert(err, 'expected error to get object with old data ' + 'locations, got success'); + next(); + }); + }, + ], + err => { + assert.ifError(err); + done(); + }, + ); + }); + + it('should not remove data locations on replayed metadata PUT', done => { let serializedNewMd; - async.waterfall([next => { - makeBackbeatRequest({ - method: 'PUT', bucket: TEST_BUCKET, - objectKey: testKey, - resourceType: 'data', - headers: { - 'content-length': testData.length, - 'x-scal-canonical-id': testArn, + async.waterfall( + [ + next => { + makeBackbeatRequest( + { + method: 'PUT', + bucket: TEST_BUCKET, + objectKey: testKey, + resourceType: 'data', + headers: { + 'content-length': testData.length, + 'x-scal-canonical-id': testArn, + }, + authCredentials: backbeatAuthCredentials, + requestBody: testData, + }, + next, + ); }, - authCredentials: backbeatAuthCredentials, - requestBody: testData }, next); - }, (response, next) => { - assert.strictEqual(response.statusCode, 200); - const newMd = Object.assign({}, testMd); - newMd.location = JSON.parse(response.body); - serializedNewMd = JSON.stringify(newMd); - async.timesSeries(2, (i, putDone) => makeBackbeatRequest({ - method: 'PUT', bucket: TEST_BUCKET, - objectKey: testKey, - resourceType: 'metadata', - queryObj: { - versionId: versionIdUtils.encode(testMd.versionId), + (response, next) => { + assert.strictEqual(response.statusCode, 200); + const newMd = Object.assign({}, testMd); + newMd.location = JSON.parse(response.body); + serializedNewMd = JSON.stringify(newMd); + async.timesSeries( + 2, + (i, putDone) => + makeBackbeatRequest( + { + method: 'PUT', + bucket: TEST_BUCKET, + objectKey: testKey, + resourceType: 'metadata', + queryObj: { + versionId: versionIdUtils.encode(testMd.versionId), + }, + authCredentials: backbeatAuthCredentials, + requestBody: serializedNewMd, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + putDone(err); + }, + ), + () => next(), + ); }, - authCredentials: backbeatAuthCredentials, - requestBody: serializedNewMd, - }, (err, response) => { + next => { + // check that the object is still readable to make + // sure we did not remove the data keys + s3.send( + new GetObjectCommand({ + Bucket: TEST_BUCKET, + Key: testKey, + }), + ) + .then(async data => { + const body = await data.Body.transformToString(); + assert.strictEqual(body, testData); + next(); + }) + .catch(err => { + next(err); + }); + }, + ], + err => { assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - putDone(err); - }), () => next()); - }, next => { - // check that the object is still readable to make - // sure we did not remove the data keys - s3.send(new GetObjectCommand({ - Bucket: TEST_BUCKET, - Key: testKey, - })).then(async data => { - const body = await data.Body.transformToString(); - assert.strictEqual(body, testData); - next(); - }).catch(err => { - next(err); - }); - }], err => { - assert.ifError(err); - done(); - }); + done(); + }, + ); }); it('should create a new version when no versionId is passed in query string', done => { let newVersion; - async.waterfall([next => { - // put object's data locations - makeBackbeatRequest({ - method: 'PUT', bucket: TEST_BUCKET, - objectKey: testKey, - resourceType: 'data', - headers: { - 'content-length': testData.length, - 'x-scal-canonical-id': testArn, + async.waterfall( + [ + next => { + // put object's data locations + makeBackbeatRequest( + { + method: 'PUT', + bucket: TEST_BUCKET, + objectKey: testKey, + resourceType: 'data', + headers: { + 'content-length': testData.length, + 'x-scal-canonical-id': testArn, + }, + authCredentials: backbeatAuthCredentials, + requestBody: testData, + }, + next, + ); }, - authCredentials: backbeatAuthCredentials, - requestBody: testData }, next); - }, (response, next) => { - assert.strictEqual(response.statusCode, 200); - // put object metadata - const oldMd = Object.assign({}, testMd); - oldMd.location = JSON.parse(response.body); - makeBackbeatRequest({ - method: 'PUT', bucket: TEST_BUCKET, - objectKey: testKey, - resourceType: 'metadata', - queryObj: { - versionId: versionIdUtils.encode(testMd.versionId), + (response, next) => { + assert.strictEqual(response.statusCode, 200); + // put object metadata + const oldMd = Object.assign({}, testMd); + oldMd.location = JSON.parse(response.body); + makeBackbeatRequest( + { + method: 'PUT', + bucket: TEST_BUCKET, + objectKey: testKey, + resourceType: 'metadata', + queryObj: { + versionId: versionIdUtils.encode(testMd.versionId), + }, + authCredentials: backbeatAuthCredentials, + requestBody: JSON.stringify(oldMd), + }, + next, + ); }, - authCredentials: backbeatAuthCredentials, - requestBody: JSON.stringify(oldMd), - }, next); - }, (response, next) => { - assert.strictEqual(response.statusCode, 200); - const parsedResponse = JSON.parse(response.body); - assert.strictEqual(parsedResponse.versionId, testMd.versionId); - // create new data locations - makeBackbeatRequest({ - method: 'PUT', bucket: TEST_BUCKET, - objectKey: testKey, - resourceType: 'data', - headers: { - 'content-length': testData.length, - 'x-scal-canonical-id': testArn, + (response, next) => { + assert.strictEqual(response.statusCode, 200); + const parsedResponse = JSON.parse(response.body); + assert.strictEqual(parsedResponse.versionId, testMd.versionId); + // create new data locations + makeBackbeatRequest( + { + method: 'PUT', + bucket: TEST_BUCKET, + objectKey: testKey, + resourceType: 'data', + headers: { + 'content-length': testData.length, + 'x-scal-canonical-id': testArn, + }, + authCredentials: backbeatAuthCredentials, + requestBody: testData, + }, + next, + ); }, - authCredentials: backbeatAuthCredentials, - requestBody: testData }, next); - }, (response, next) => { - assert.strictEqual(response.statusCode, 200); - // create a new version with the new data locations, - // not passing 'versionId' in the query string - const newMd = Object.assign({}, testMd); - newMd.location = JSON.parse(response.body); - makeBackbeatRequest({ - method: 'PUT', bucket: TEST_BUCKET, - objectKey: testKey, - resourceType: 'metadata', - authCredentials: backbeatAuthCredentials, - requestBody: JSON.stringify(newMd), - }, next); - }, (response, next) => { - assert.strictEqual(response.statusCode, 200); - const parsedResponse = JSON.parse(response.body); - newVersion = parsedResponse.versionId; - assert.notStrictEqual(newVersion, testMd.versionId); - // give some time for the async deletes to complete, - // then check that we can read the latest version - setTimeout(() => s3.send(new GetObjectCommand({ - Bucket: TEST_BUCKET, - Key: testKey, - })).then(async data => { - const body = await data.Body.transformToString(); - assert.strictEqual(body, testData); - next(); - }).catch(err => { - next(err); - }), 1000); - }, next => { - // check that the previous object version is still readable - s3.send(new GetObjectCommand({ - Bucket: TEST_BUCKET, - Key: testKey, - VersionId: versionIdUtils.encode(testMd.versionId), - })).then(async data => { - const body = await data.Body.transformToString(); - assert.strictEqual(body, testData); - next(); - }).catch(err => { - next(err); - }); - }], err => { - assert.ifError(err); - done(); - }); + (response, next) => { + assert.strictEqual(response.statusCode, 200); + // create a new version with the new data locations, + // not passing 'versionId' in the query string + const newMd = Object.assign({}, testMd); + newMd.location = JSON.parse(response.body); + makeBackbeatRequest( + { + method: 'PUT', + bucket: TEST_BUCKET, + objectKey: testKey, + resourceType: 'metadata', + authCredentials: backbeatAuthCredentials, + requestBody: JSON.stringify(newMd), + }, + next, + ); + }, + (response, next) => { + assert.strictEqual(response.statusCode, 200); + const parsedResponse = JSON.parse(response.body); + newVersion = parsedResponse.versionId; + assert.notStrictEqual(newVersion, testMd.versionId); + // give some time for the async deletes to complete, + // then check that we can read the latest version + setTimeout( + () => + s3 + .send( + new GetObjectCommand({ + Bucket: TEST_BUCKET, + Key: testKey, + }), + ) + .then(async data => { + const body = await data.Body.transformToString(); + assert.strictEqual(body, testData); + next(); + }) + .catch(err => { + next(err); + }), + 1000, + ); + }, + next => { + // check that the previous object version is still readable + s3.send( + new GetObjectCommand({ + Bucket: TEST_BUCKET, + Key: testKey, + VersionId: versionIdUtils.encode(testMd.versionId), + }), + ) + .then(async data => { + const body = await data.Body.transformToString(); + assert.strictEqual(body, testData); + next(); + }) + .catch(err => { + next(err); + }); + }, + ], + err => { + assert.ifError(err); + done(); + }, + ); }); }); describe('backbeat authorization checks', () => { const { accessKeyId: accessKeyLisa, secretAccessKey: secretAccessKeyLisa } = getCredentials('lisa'); - [{ method: 'PUT', resourceType: 'metadata' }, - { method: 'PUT', resourceType: 'data' }].forEach(test => { - const queryObj = test.resourceType === 'data' ? { v2: '' } : {}; - it(`${test.method} ${test.resourceType} should respond with ` + - '403 Forbidden if no credentials are provided', - done => { - makeBackbeatRequest({ - method: test.method, bucket: TEST_BUCKET, - objectKey: TEST_KEY, resourceType: test.resourceType, - queryObj, - }, - err => { - assert(err); - assert.strictEqual(err.statusCode, 403); - assert.strictEqual(err.code, 'AccessDenied'); - done(); - }); - }); - it(`${test.method} ${test.resourceType} should respond with ` + - '403 Forbidden if wrong credentials are provided', + [ + { method: 'PUT', resourceType: 'metadata' }, + { method: 'PUT', resourceType: 'data' }, + ].forEach(test => { + const queryObj = test.resourceType === 'data' ? { v2: '' } : {}; + it( + `${test.method} ${test.resourceType} should respond with ` + + '403 Forbidden if no credentials are provided', done => { - makeBackbeatRequest({ - method: test.method, bucket: TEST_BUCKET, - objectKey: TEST_KEY, resourceType: test.resourceType, - queryObj, - authCredentials: { - accessKey: 'wrong', - secretKey: 'still wrong', + makeBackbeatRequest( + { + method: test.method, + bucket: TEST_BUCKET, + objectKey: TEST_KEY, + resourceType: test.resourceType, + queryObj, }, - }, - err => { - assert(err); - assert.strictEqual(err.statusCode, 403); - assert.strictEqual(err.code, 'InvalidAccessKeyId'); - done(); - }); - }); - it(`${test.method} ${test.resourceType} should respond with ` + - '403 Forbidden if the account does not match the ' + - 'backbeat user', + err => { + assert(err); + assert.strictEqual(err.statusCode, 403); + assert.strictEqual(err.code, 'AccessDenied'); + done(); + }, + ); + }, + ); + it( + `${test.method} ${test.resourceType} should respond with ` + + '403 Forbidden if wrong credentials are provided', done => { - makeBackbeatRequest({ - method: test.method, bucket: TEST_BUCKET, - objectKey: TEST_KEY, resourceType: test.resourceType, - queryObj, - authCredentials: { - accessKey: accessKeyLisa, - secretKey: secretAccessKeyLisa, + makeBackbeatRequest( + { + method: test.method, + bucket: TEST_BUCKET, + objectKey: TEST_KEY, + resourceType: test.resourceType, + queryObj, + authCredentials: { + accessKey: 'wrong', + secretKey: 'still wrong', + }, }, - }, - err => { - assert(err); - assert.strictEqual(err.statusCode, 403); - assert.strictEqual(err.code, 'AccessDenied'); - done(); - }); - }); - it(`${test.method} ${test.resourceType} should respond with ` + - '403 Forbidden if backbeat user has wrong secret key', + err => { + assert(err); + assert.strictEqual(err.statusCode, 403); + assert.strictEqual(err.code, 'InvalidAccessKeyId'); + done(); + }, + ); + }, + ); + it( + `${test.method} ${test.resourceType} should respond with ` + + '403 Forbidden if the account does not match the ' + + 'backbeat user', done => { - makeBackbeatRequest({ - method: test.method, bucket: TEST_BUCKET, - objectKey: TEST_KEY, resourceType: test.resourceType, - queryObj, - authCredentials: { - accessKey: backbeatAuthCredentials.accessKey, - secretKey: 'hastalavista', + makeBackbeatRequest( + { + method: test.method, + bucket: TEST_BUCKET, + objectKey: TEST_KEY, + resourceType: test.resourceType, + queryObj, + authCredentials: { + accessKey: accessKeyLisa, + secretKey: secretAccessKeyLisa, + }, }, - }, - err => { - assert(err); - assert.strictEqual(err.statusCode, 403); - assert.strictEqual(err.code, 'SignatureDoesNotMatch'); - done(); - }); - }); - }); + err => { + assert(err); + assert.strictEqual(err.statusCode, 403); + assert.strictEqual(err.code, 'AccessDenied'); + done(); + }, + ); + }, + ); + it( + `${test.method} ${test.resourceType} should respond with ` + + '403 Forbidden if backbeat user has wrong secret key', + done => { + makeBackbeatRequest( + { + method: test.method, + bucket: TEST_BUCKET, + objectKey: TEST_KEY, + resourceType: test.resourceType, + queryObj, + authCredentials: { + accessKey: backbeatAuthCredentials.accessKey, + secretKey: 'hastalavista', + }, + }, + err => { + assert(err); + assert.strictEqual(err.statusCode, 403); + assert.strictEqual(err.code, 'SignatureDoesNotMatch'); + done(); + }, + ); + }, + ); + }); const apiProxy = !!config.backbeat; describe(`when api proxy is ${apiProxy ? '' : 'NOT '}configured`, () => { @@ -2939,271 +3938,325 @@ describe('backbeat routes', () => { it(`GET /_/backbeat/api/... should respond with ${ apiProxy ? 503 : 405 - } on authenticated requests (API server down)`, - done => { - const options = { - authCredentials: { - accessKey: accessKeyLisa, - secretKey: secretAccessKeyLisa, - }, - hostname: ipAddress, - port: 8000, - method: 'GET', - path: '/_/backbeat/api/crr/failed', - jsonResponse: true, - }; - makeRequest(options, err => { - assert(err); - const expected = apiProxy ? 503 : 405; - assert.strictEqual(err.statusCode, expected); - assert.strictEqual(err.code, errors[expected]); - done(); - }); + } on authenticated requests (API server down)`, done => { + const options = { + authCredentials: { + accessKey: accessKeyLisa, + secretKey: secretAccessKeyLisa, + }, + hostname: ipAddress, + port: 8000, + method: 'GET', + path: '/_/backbeat/api/crr/failed', + jsonResponse: true, + }; + makeRequest(options, err => { + assert(err); + const expected = apiProxy ? 503 : 405; + assert.strictEqual(err.statusCode, expected); + assert.strictEqual(err.code, errors[expected]); + done(); }); + }); it(`GET /_/backbeat/api/... should respond with ${ apiProxy ? 403 : 405 - } if the request is unauthenticated`, - done => { - const options = { - hostname: ipAddress, - port: 8000, - method: 'GET', - path: '/_/backbeat/api/crr/failed', - jsonResponse: true, - }; - makeRequest(options, err => { - assert(err); - const expected = apiProxy ? 403 : 405; - assert.strictEqual(err.statusCode, expected); - assert.strictEqual(err.code, errors[expected]); - done(); - }); + } if the request is unauthenticated`, done => { + const options = { + hostname: ipAddress, + port: 8000, + method: 'GET', + path: '/_/backbeat/api/crr/failed', + jsonResponse: true, + }; + makeRequest(options, err => { + assert(err); + const expected = apiProxy ? 403 : 405; + assert.strictEqual(err.statusCode, expected); + assert.strictEqual(err.code, errors[expected]); + done(); }); + }); }); }); describe('GET Metadata route', () => { - beforeEach(done => makeBackbeatRequest({ - method: 'PUT', bucket: TEST_BUCKET, - objectKey: TEST_KEY, - resourceType: 'metadata', - queryObj: { - versionId: versionIdUtils.encode(testMd.versionId), - }, - authCredentials: backbeatAuthCredentials, - requestBody: JSON.stringify(testMd), - }, done)); + beforeEach(done => + makeBackbeatRequest( + { + method: 'PUT', + bucket: TEST_BUCKET, + objectKey: TEST_KEY, + resourceType: 'metadata', + queryObj: { + versionId: versionIdUtils.encode(testMd.versionId), + }, + authCredentials: backbeatAuthCredentials, + requestBody: JSON.stringify(testMd), + }, + done, + ), + ); it('should return metadata blob for a versionId', done => { - makeBackbeatRequest({ - method: 'GET', bucket: TEST_BUCKET, - objectKey: TEST_KEY, resourceType: 'metadata', - authCredentials: backbeatAuthCredentials, - queryObj: { - versionId: versionIdUtils.encode(testMd.versionId), + makeBackbeatRequest( + { + method: 'GET', + bucket: TEST_BUCKET, + objectKey: TEST_KEY, + resourceType: 'metadata', + authCredentials: backbeatAuthCredentials, + queryObj: { + versionId: versionIdUtils.encode(testMd.versionId), + }, }, - }, (err, data) => { - assert.ifError(err); - const parsedBody = JSON.parse(JSON.parse(data.body).Body); - assert.strictEqual(data.statusCode, 200); - assert.deepStrictEqual(parsedBody, testMd); - done(); - }); + (err, data) => { + assert.ifError(err); + const parsedBody = JSON.parse(JSON.parse(data.body).Body); + assert.strictEqual(data.statusCode, 200); + assert.deepStrictEqual(parsedBody, testMd); + done(); + }, + ); }); it('should return error if bucket does not exist', done => { - makeBackbeatRequest({ - method: 'GET', bucket: 'blah', - objectKey: TEST_KEY, resourceType: 'metadata', - authCredentials: backbeatAuthCredentials, - queryObj: { - versionId: versionIdUtils.encode(testMd.versionId), + makeBackbeatRequest( + { + method: 'GET', + bucket: 'blah', + objectKey: TEST_KEY, + resourceType: 'metadata', + authCredentials: backbeatAuthCredentials, + queryObj: { + versionId: versionIdUtils.encode(testMd.versionId), + }, }, - }, (err, data) => { - assert.strictEqual(data.statusCode, 404); - const body = JSON.parse(data.body); - assert.strictEqual(body.code, 'NoSuchBucket'); - // err is parsed data.body + statusCode - assert.deepStrictEqual(err, { ...body, statusCode: data.statusCode }); - done(); - }); + (err, data) => { + assert.strictEqual(data.statusCode, 404); + const body = JSON.parse(data.body); + assert.strictEqual(body.code, 'NoSuchBucket'); + // err is parsed data.body + statusCode + assert.deepStrictEqual(err, { ...body, statusCode: data.statusCode }); + done(); + }, + ); }); it('should return error if object does not exist', done => { - makeBackbeatRequest({ - method: 'GET', bucket: TEST_BUCKET, - objectKey: 'blah', resourceType: 'metadata', - authCredentials: backbeatAuthCredentials, - queryObj: { - versionId: versionIdUtils.encode(testMd.versionId), + makeBackbeatRequest( + { + method: 'GET', + bucket: TEST_BUCKET, + objectKey: 'blah', + resourceType: 'metadata', + authCredentials: backbeatAuthCredentials, + queryObj: { + versionId: versionIdUtils.encode(testMd.versionId), + }, }, - }, (err, data) => { - assert.strictEqual(data.statusCode, 404); - const body = JSON.parse(data.body); - assert.strictEqual(body.code, 'ObjNotFound'); - // err is parsed data.body + statusCode - assert.deepStrictEqual(err, { ...body, statusCode: data.statusCode }); - done(); - }); + (err, data) => { + assert.strictEqual(data.statusCode, 404); + const body = JSON.parse(data.body); + assert.strictEqual(body.code, 'ObjNotFound'); + // err is parsed data.body + statusCode + assert.deepStrictEqual(err, { ...body, statusCode: data.statusCode }); + done(); + }, + ); }); }); describeIfLocationAws('backbeat multipart upload operations (external location)', function test() { this.timeout(10000); - it('should put tags if the source is AWS and tags are ' + - 'provided when initiating the multipart upload', done => { - const awsKey = uuidv4(); - const multipleBackendPath = - `/_/backbeat/multiplebackenddata/${awsBucket}/${awsKey}`; - let uploadId; - let partData; - async.series([ - next => - makeRequest({ - authCredentials: backbeatAuthCredentials, - hostname: ipAddress, - port: 8000, - method: 'POST', - path: multipleBackendPath, - queryObj: { operation: 'initiatempu' }, - headers: { - 'x-scal-storage-class': awsLocation, - 'x-scal-storage-type': 'aws_s3', - 'x-scal-tags': JSON.stringify({ 'key1': 'value1' }), - }, - jsonResponse: true, - }, (err, data) => { - if (err) { - return next(err); - } - uploadId = JSON.parse(data.body).uploadId; - return next(); - }), - next => - makeRequest({ - authCredentials: backbeatAuthCredentials, - hostname: ipAddress, - port: 8000, - method: 'PUT', - path: multipleBackendPath, - queryObj: { operation: 'putpart' }, - headers: { - 'x-scal-storage-class': awsLocation, - 'x-scal-storage-type': 'aws_s3', - 'x-scal-upload-id': uploadId, - 'x-scal-part-number': '1', - 'content-length': testData.length, - }, - requestBody: testData, - jsonResponse: true, - }, (err, data) => { - if (err) { - return next(err); - } - const body = JSON.parse(data.body); - partData = [{ - PartNumber: [body.partNumber], - ETag: [body.ETag], - }]; - return next(); - }), - next => - makeRequest({ - authCredentials: backbeatAuthCredentials, - hostname: ipAddress, - port: 8000, - method: 'POST', - path: multipleBackendPath, - queryObj: { operation: 'completempu' }, - headers: { - 'x-scal-storage-class': awsLocation, - 'x-scal-storage-type': 'aws_s3', - 'x-scal-upload-id': uploadId, - }, - requestBody: JSON.stringify(partData), - jsonResponse: true, - }, next), - next => - awsClient.send(new GetObjectTaggingCommand({ - Bucket: awsBucket, - Key: awsKey, - }), (err, data) => { - assert.ifError(err); - assert.deepStrictEqual(data.TagSet, [{ - Key: 'key1', - Value: 'value1', - }]); - next(); - }), - ], done); - }); + it( + 'should put tags if the source is AWS and tags are ' + 'provided when initiating the multipart upload', + done => { + const awsKey = uuidv4(); + const multipleBackendPath = `/_/backbeat/multiplebackenddata/${awsBucket}/${awsKey}`; + let uploadId; + let partData; + async.series( + [ + next => + makeRequest( + { + authCredentials: backbeatAuthCredentials, + hostname: ipAddress, + port: 8000, + method: 'POST', + path: multipleBackendPath, + queryObj: { operation: 'initiatempu' }, + headers: { + 'x-scal-storage-class': awsLocation, + 'x-scal-storage-type': 'aws_s3', + 'x-scal-tags': JSON.stringify({ key1: 'value1' }), + }, + jsonResponse: true, + }, + (err, data) => { + if (err) { + return next(err); + } + uploadId = JSON.parse(data.body).uploadId; + return next(); + }, + ), + next => + makeRequest( + { + authCredentials: backbeatAuthCredentials, + hostname: ipAddress, + port: 8000, + method: 'PUT', + path: multipleBackendPath, + queryObj: { operation: 'putpart' }, + headers: { + 'x-scal-storage-class': awsLocation, + 'x-scal-storage-type': 'aws_s3', + 'x-scal-upload-id': uploadId, + 'x-scal-part-number': '1', + 'content-length': testData.length, + }, + requestBody: testData, + jsonResponse: true, + }, + (err, data) => { + if (err) { + return next(err); + } + const body = JSON.parse(data.body); + partData = [ + { + PartNumber: [body.partNumber], + ETag: [body.ETag], + }, + ]; + return next(); + }, + ), + next => + makeRequest( + { + authCredentials: backbeatAuthCredentials, + hostname: ipAddress, + port: 8000, + method: 'POST', + path: multipleBackendPath, + queryObj: { operation: 'completempu' }, + headers: { + 'x-scal-storage-class': awsLocation, + 'x-scal-storage-type': 'aws_s3', + 'x-scal-upload-id': uploadId, + }, + requestBody: JSON.stringify(partData), + jsonResponse: true, + }, + next, + ), + next => + awsClient.send( + new GetObjectTaggingCommand({ + Bucket: awsBucket, + Key: awsKey, + }), + (err, data) => { + assert.ifError(err); + assert.deepStrictEqual(data.TagSet, [ + { + Key: 'key1', + Value: 'value1', + }, + ]); + next(); + }, + ), + ], + done, + ); + }, + ); - it('should put tags if the source is Azure and tags are provided ' + - 'when completing the multipart upload', done => { - const containerName = getAzureContainerName(azureLocation); - const blob = uuidv4(); - const multipleBackendPath = - `/_/backbeat/multiplebackenddata/${containerName}/${blob}`; - const uploadId = uuidv4().replace(/-/g, ''); - let partData; - async.series([ - next => - makeRequest({ - authCredentials: backbeatAuthCredentials, - hostname: ipAddress, - port: 8000, - method: 'PUT', - path: multipleBackendPath, - queryObj: { operation: 'putpart' }, - headers: { - 'x-scal-storage-class': azureLocation, - 'x-scal-storage-type': 'azure', - 'x-scal-upload-id': uploadId, - 'x-scal-part-number': '1', - 'content-length': testData.length, - }, - requestBody: testData, - jsonResponse: true, - }, (err, data) => { - if (err) { - return next(err); - } - const body = JSON.parse(data.body); - partData = [{ - PartNumber: [body.partNumber], - ETag: [body.ETag], - NumberSubParts: [body.numberSubParts], - }]; - return next(); - }), - next => - makeRequest({ - authCredentials: backbeatAuthCredentials, - hostname: ipAddress, - port: 8000, - method: 'POST', - path: multipleBackendPath, - queryObj: { operation: 'completempu' }, - headers: { - 'x-scal-storage-class': azureLocation, - 'x-scal-storage-type': 'azure', - 'x-scal-upload-id': uploadId, - 'x-scal-tags': JSON.stringify({ 'key1': 'value1' }), - }, - requestBody: JSON.stringify(partData), - jsonResponse: true, - }, next), - next => - azureClient.getContainerClient(containerName).getBlobClient(blob).getProperties() - .then(result => { - const tags = JSON.parse(result.metadata.tags); - assert.deepStrictEqual(tags, { key1: 'value1' }); - return next(); - }, next), - ], done); - }); + it( + 'should put tags if the source is Azure and tags are provided ' + 'when completing the multipart upload', + done => { + const containerName = getAzureContainerName(azureLocation); + const blob = uuidv4(); + const multipleBackendPath = `/_/backbeat/multiplebackenddata/${containerName}/${blob}`; + const uploadId = uuidv4().replace(/-/g, ''); + let partData; + async.series( + [ + next => + makeRequest( + { + authCredentials: backbeatAuthCredentials, + hostname: ipAddress, + port: 8000, + method: 'PUT', + path: multipleBackendPath, + queryObj: { operation: 'putpart' }, + headers: { + 'x-scal-storage-class': azureLocation, + 'x-scal-storage-type': 'azure', + 'x-scal-upload-id': uploadId, + 'x-scal-part-number': '1', + 'content-length': testData.length, + }, + requestBody: testData, + jsonResponse: true, + }, + (err, data) => { + if (err) { + return next(err); + } + const body = JSON.parse(data.body); + partData = [ + { + PartNumber: [body.partNumber], + ETag: [body.ETag], + NumberSubParts: [body.numberSubParts], + }, + ]; + return next(); + }, + ), + next => + makeRequest( + { + authCredentials: backbeatAuthCredentials, + hostname: ipAddress, + port: 8000, + method: 'POST', + path: multipleBackendPath, + queryObj: { operation: 'completempu' }, + headers: { + 'x-scal-storage-class': azureLocation, + 'x-scal-storage-type': 'azure', + 'x-scal-upload-id': uploadId, + 'x-scal-tags': JSON.stringify({ key1: 'value1' }), + }, + requestBody: JSON.stringify(partData), + jsonResponse: true, + }, + next, + ), + next => + azureClient + .getContainerClient(containerName) + .getBlobClient(blob) + .getProperties() + .then(result => { + const tags = JSON.parse(result.metadata.tags); + assert.deepStrictEqual(tags, { key1: 'value1' }); + return next(); + }, next), + ], + done, + ); + }, + ); }); describe('Batch Delete Route', function test() { this.timeout(30000); @@ -3212,430 +4265,559 @@ describe('backbeat routes', () => { let location; const testKey = 'batch-delete-test-key'; - async.series([ - done => { - s3.send(new PutObjectCommand({ - Bucket: TEST_BUCKET, - Key: testKey, - Body: Buffer.from('hello'), - })).then(data => { - versionId = data.VersionId; - done(); - }).catch(err => { - done(err); - }); - }, - done => { - makeBackbeatRequest({ - method: 'GET', - bucket: TEST_BUCKET, - objectKey: testKey, - resourceType: 'metadata', - authCredentials: backbeatAuthCredentials, - queryObj: { - versionId, - }, - }, (err, data) => { - assert.ifError(err); - assert.strictEqual(data.statusCode, 200); - const metadata = JSON.parse( - JSON.parse(data.body).Body); - location = metadata.location; - done(); - }); - }, - done => { - const options = { - authCredentials: backbeatAuthCredentials, - hostname: ipAddress, - port: 8000, - method: 'POST', - path: `/_/backbeat/batchdelete/${TEST_BUCKET}/${testKey}`, - requestBody: - `{"Locations":${JSON.stringify(location)}}`, - jsonResponse: true, - }; - makeRequest(options, done); - }, - done => { - s3.send(new GetObjectCommand({ - Bucket: TEST_BUCKET, - Key: testKey, - })).then(() => { - done(new Error('Expected error')); - }).catch(err => { - // should error out as location shall no longer exist - assert(err); - assert.strictEqual(err.$metadata.httpStatusCode, 503); - done(); - }); - }, - ], done); + async.series( + [ + done => { + s3.send( + new PutObjectCommand({ + Bucket: TEST_BUCKET, + Key: testKey, + Body: Buffer.from('hello'), + }), + ) + .then(data => { + versionId = data.VersionId; + done(); + }) + .catch(err => { + done(err); + }); + }, + done => { + makeBackbeatRequest( + { + method: 'GET', + bucket: TEST_BUCKET, + objectKey: testKey, + resourceType: 'metadata', + authCredentials: backbeatAuthCredentials, + queryObj: { + versionId, + }, + }, + (err, data) => { + assert.ifError(err); + assert.strictEqual(data.statusCode, 200); + const metadata = JSON.parse(JSON.parse(data.body).Body); + location = metadata.location; + done(); + }, + ); + }, + done => { + const options = { + authCredentials: backbeatAuthCredentials, + hostname: ipAddress, + port: 8000, + method: 'POST', + path: `/_/backbeat/batchdelete/${TEST_BUCKET}/${testKey}`, + requestBody: `{"Locations":${JSON.stringify(location)}}`, + jsonResponse: true, + }; + makeRequest(options, done); + }, + done => { + s3.send( + new GetObjectCommand({ + Bucket: TEST_BUCKET, + Key: testKey, + }), + ) + .then(() => { + done(new Error('Expected error')); + }) + .catch(err => { + // should error out as location shall no longer exist + assert(err); + assert.strictEqual(err.$metadata.httpStatusCode, 503); + done(); + }); + }, + ], + done, + ); }); itIfLocationAws('should batch delete a versioned AWS location', done => { let versionId; const awsKey = `${TEST_BUCKET}/batch-delete-test-key-${makeid(8)}`; - async.series([ - done => { - awsClient.send(new PutObjectCommand({ - Bucket: awsBucket, - Key: awsKey, - Body: Buffer.from('hello'), - })).then(data => { - versionId = data.VersionId; - done(); - }).catch(err => { - done(err); - }); - }, - done => { - const location = [{ - key: awsKey, - size: 5, - dataStoreName: awsLocation, - dataStoreVersionId: versionId, - }]; - const reqBody = `{"Locations":${JSON.stringify(location)}}`; - const options = { - authCredentials: backbeatAuthCredentials, - hostname: ipAddress, - port: 8000, - method: 'POST', - path: '/_/backbeat/batchdelete', - requestBody: reqBody, - jsonResponse: true, - }; - makeRequest(options, done); - }, - done => { - awsClient.send(new GetObjectCommand({ - Bucket: awsBucket, - Key: awsKey, - })).then(() => { - done(new Error('Expected error')); - }).catch(err => { - // should error out as location shall no longer exist - assert(err); - done(); - }); - }, - ], done); + async.series( + [ + done => { + awsClient + .send( + new PutObjectCommand({ + Bucket: awsBucket, + Key: awsKey, + Body: Buffer.from('hello'), + }), + ) + .then(data => { + versionId = data.VersionId; + done(); + }) + .catch(err => { + done(err); + }); + }, + done => { + const location = [ + { + key: awsKey, + size: 5, + dataStoreName: awsLocation, + dataStoreVersionId: versionId, + }, + ]; + const reqBody = `{"Locations":${JSON.stringify(location)}}`; + const options = { + authCredentials: backbeatAuthCredentials, + hostname: ipAddress, + port: 8000, + method: 'POST', + path: '/_/backbeat/batchdelete', + requestBody: reqBody, + jsonResponse: true, + }; + makeRequest(options, done); + }, + done => { + awsClient + .send( + new GetObjectCommand({ + Bucket: awsBucket, + Key: awsKey, + }), + ) + .then(() => { + done(new Error('Expected error')); + }) + .catch(err => { + // should error out as location shall no longer exist + assert(err); + done(); + }); + }, + ], + done, + ); }); it('should fail with error if given malformed JSON', done => { - async.series([ - done => { - const options = { - authCredentials: backbeatAuthCredentials, - hostname: ipAddress, - port: 8000, - method: 'POST', - path: '/_/backbeat/batchdelete', - requestBody: 'NOTJSON', - jsonResponse: true, - }; - makeRequest(options, done); + async.series( + [ + done => { + const options = { + authCredentials: backbeatAuthCredentials, + hostname: ipAddress, + port: 8000, + method: 'POST', + path: '/_/backbeat/batchdelete', + requestBody: 'NOTJSON', + jsonResponse: true, + }; + makeRequest(options, done); + }, + ], + err => { + assert(err); + done(); }, - ], err => { - assert(err); - done(); - }); + ); }); // TODO: unskip test when S3C-9123 is fixed itSkipS3C('should skip batch delete of a non-existent location', done => { - async.series([ - done => { - const options = { - authCredentials: backbeatAuthCredentials, - hostname: ipAddress, - port: 8000, - method: 'POST', - path: '/_/backbeat/batchdelete', - requestBody: - '{"Locations":' + - '[{"key":"abcdef","dataStoreName":"us-east-1"}]}', - jsonResponse: true, - }; - makeRequest(options, done); - }, - ], done); + async.series( + [ + done => { + const options = { + authCredentials: backbeatAuthCredentials, + hostname: ipAddress, + port: 8000, + method: 'POST', + path: '/_/backbeat/batchdelete', + requestBody: '{"Locations":' + '[{"key":"abcdef","dataStoreName":"us-east-1"}]}', + jsonResponse: true, + }; + makeRequest(options, done); + }, + ], + done, + ); }); it('should skip batch delete of empty location array', done => { - async.series([ - done => { - const options = { - authCredentials: backbeatAuthCredentials, - hostname: ipAddress, - port: 8000, - method: 'POST', - path: '/_/backbeat/batchdelete', - requestBody: '{"Locations":[]}', - jsonResponse: true, - }; - makeRequest(options, done); - }, - ], done); + async.series( + [ + done => { + const options = { + authCredentials: backbeatAuthCredentials, + hostname: ipAddress, + port: 8000, + method: 'POST', + path: '/_/backbeat/batchdelete', + requestBody: '{"Locations":[]}', + jsonResponse: true, + }; + makeRequest(options, done); + }, + ], + done, + ); }); - itIfLocationAws('should not put delete tags if the source is not Azure and ' + - 'if-unmodified-since header is not provided', done => { - const awsKey = uuidv4(); - async.series([ - next => { - awsClient.send(new PutObjectCommand({ - Bucket: awsBucket, - Key: awsKey, - })).then(result => { - next(null, result); - }).catch(err => { - next(err); - }); - }, - next => - makeRequest({ - authCredentials: backbeatAuthCredentials, - hostname: ipAddress, - port: 8000, - method: 'POST', - path: '/_/backbeat/batchdelete', - headers: { - 'x-scal-storage-class': awsLocation, - 'x-scal-tags': JSON.stringify({ - 'scal-delete-marker': 'true', - 'scal-delete-service': 'lifecycle-transition', - }), + itIfLocationAws( + 'should not put delete tags if the source is not Azure and ' + 'if-unmodified-since header is not provided', + done => { + const awsKey = uuidv4(); + async.series( + [ + next => { + awsClient + .send( + new PutObjectCommand({ + Bucket: awsBucket, + Key: awsKey, + }), + ) + .then(result => { + next(null, result); + }) + .catch(err => { + next(err); + }); }, - requestBody: JSON.stringify({ - Locations: [{ - key: awsKey, - dataStoreName: awsLocation, - }], - }), - jsonResponse: true, - }, next), - next => { - awsClient.send(new GetObjectTaggingCommand({ - Bucket: awsBucket, - Key: awsKey, - })).then(data => { - assert.deepStrictEqual(data.TagSet, []); - next(null, data); - }).catch(err => { - next(err); - }); - }, - ], done); - }); - - itIfLocationAws('should not put tags if the source is not Azure and ' + - 'if-unmodified-since condition is not met', done => { - const awsKey = uuidv4(); - async.series([ - next => - awsClient.send(new PutObjectCommand({ - Bucket: awsBucket, - Key: awsKey, - })).then(result => next(null, result)).catch(err => next(err)), - next => - makeRequest({ - authCredentials: backbeatAuthCredentials, - hostname: ipAddress, - port: 8000, - method: 'POST', - path: '/_/backbeat/batchdelete', - headers: { - 'if-unmodified-since': - 'Sun, 31 Mar 2019 00:00:00 GMT', - 'x-scal-storage-class': awsLocation, - 'x-scal-tags': JSON.stringify({ - 'scal-delete-marker': 'true', - 'scal-delete-service': 'lifecycle-transition', - }), + next => + makeRequest( + { + authCredentials: backbeatAuthCredentials, + hostname: ipAddress, + port: 8000, + method: 'POST', + path: '/_/backbeat/batchdelete', + headers: { + 'x-scal-storage-class': awsLocation, + 'x-scal-tags': JSON.stringify({ + 'scal-delete-marker': 'true', + 'scal-delete-service': 'lifecycle-transition', + }), + }, + requestBody: JSON.stringify({ + Locations: [ + { + key: awsKey, + dataStoreName: awsLocation, + }, + ], + }), + jsonResponse: true, + }, + next, + ), + next => { + awsClient + .send( + new GetObjectTaggingCommand({ + Bucket: awsBucket, + Key: awsKey, + }), + ) + .then(data => { + assert.deepStrictEqual(data.TagSet, []); + next(null, data); + }) + .catch(err => { + next(err); + }); }, - requestBody: JSON.stringify({ - Locations: [{ - key: awsKey, - dataStoreName: awsLocation, - }], - }), - jsonResponse: true, - }, next), - next => { - awsClient.send(new GetObjectTaggingCommand({ - Bucket: awsBucket, - Key: awsKey, - })).then(data => { - assert.deepStrictEqual(data.TagSet, []); - next(); - }).catch(err => { - next(err); - }); - }, - ], done); - }); + ], + done, + ); + }, + ); - itIfLocationAws('should put tags if the source is not Azure and ' + - 'if-unmodified-since condition is met', done => { - const awsKey = uuidv4(); - let lastModified; - async.series([ - next => - awsClient.send(new PutObjectCommand({ - Bucket: awsBucket, - Key: awsKey, - })).then(result => next(null, result)).catch(err => next(err)), - next => - awsClient.send(new HeadObjectCommand({ - Bucket: awsBucket, - Key: awsKey, - })).then(data => { - lastModified = data.LastModified; - next(null, data); - }).catch(err => next(err)), - next => - makeRequest({ - authCredentials: backbeatAuthCredentials, - hostname: ipAddress, - port: 8000, - method: 'POST', - path: `/_/backbeat/batchdelete/${awsBucket}/${awsKey}`, - headers: { - 'if-unmodified-since': lastModified, - 'x-scal-storage-class': awsLocation, - 'x-scal-tags': JSON.stringify({ - 'scal-delete-marker': 'true', - 'scal-delete-service': 'lifecycle-transition', - }), + itIfLocationAws( + 'should not put tags if the source is not Azure and ' + 'if-unmodified-since condition is not met', + done => { + const awsKey = uuidv4(); + async.series( + [ + next => + awsClient + .send( + new PutObjectCommand({ + Bucket: awsBucket, + Key: awsKey, + }), + ) + .then(result => next(null, result)) + .catch(err => next(err)), + next => + makeRequest( + { + authCredentials: backbeatAuthCredentials, + hostname: ipAddress, + port: 8000, + method: 'POST', + path: '/_/backbeat/batchdelete', + headers: { + 'if-unmodified-since': 'Sun, 31 Mar 2019 00:00:00 GMT', + 'x-scal-storage-class': awsLocation, + 'x-scal-tags': JSON.stringify({ + 'scal-delete-marker': 'true', + 'scal-delete-service': 'lifecycle-transition', + }), + }, + requestBody: JSON.stringify({ + Locations: [ + { + key: awsKey, + dataStoreName: awsLocation, + }, + ], + }), + jsonResponse: true, + }, + next, + ), + next => { + awsClient + .send( + new GetObjectTaggingCommand({ + Bucket: awsBucket, + Key: awsKey, + }), + ) + .then(data => { + assert.deepStrictEqual(data.TagSet, []); + next(); + }) + .catch(err => { + next(err); + }); }, - requestBody: JSON.stringify({ - Locations: [{ - key: awsKey, - dataStoreName: awsLocation, - }], - }), - jsonResponse: true, - }, next), - next => - awsClient.send(new GetObjectTaggingCommand({ - Bucket: awsBucket, - Key: awsKey, - })).then(data => { - assert.strictEqual(data.TagSet.length, 2); - data.TagSet.forEach(tag => { - const { Key, Value } = tag; - const isValidTag = - Key === 'scal-delete-marker' || - Key === 'scal-delete-service'; - assert(isValidTag); - if (Key === 'scal-delete-marker') { - assert.strictEqual(Value, 'true'); - } - if (Key === 'scal-delete-service') { - assert.strictEqual( - Value, 'lifecycle-transition'); - } - }); - next(null, data); - }).catch(err => { - assert.ifError(err); - next(err); - }), - ], done); - }); + ], + done, + ); + }, + ); - itIfLocationAzure('should not delete the object if the source is Azure and ' + - 'if-unmodified-since condition is not met', done => { - const blob = uuidv4(); - async.series([ - next => - azureClient.getContainerClient(containerName).uploadBlockBlob(blob, 'a', 1) - .then(() => next(), next), - next => - makeRequest({ - authCredentials: backbeatAuthCredentials, - hostname: ipAddress, - port: 8000, - - method: 'POST', - path: - `/_/backbeat/batchdelete/${containerName}/${blob}`, - headers: { - 'if-unmodified-since': - 'Sun, 31 Mar 2019 00:00:00 GMT', - 'x-scal-storage-class': azureLocation, - 'x-scal-tags': JSON.stringify({ - 'scal-delete-marker': 'true', - 'scal-delete-service': 'lifecycle-transition', - }), - }, - requestBody: JSON.stringify({ - Locations: [{ - key: blob, - dataStoreName: azureLocation, - }], - }), - jsonResponse: true, - }, err => { - if (err && err.statusCode === 412) { - return next(); - } - return next(err); - }), - next => - azureClient.getContainerClient(containerName).getBlobClient(blob).getProperties() - .then(result => { - assert(result); - return next(); - }, next), - ], done); - }); + itIfLocationAws( + 'should put tags if the source is not Azure and ' + 'if-unmodified-since condition is met', + done => { + const awsKey = uuidv4(); + let lastModified; + async.series( + [ + next => + awsClient + .send( + new PutObjectCommand({ + Bucket: awsBucket, + Key: awsKey, + }), + ) + .then(result => next(null, result)) + .catch(err => next(err)), + next => + awsClient + .send( + new HeadObjectCommand({ + Bucket: awsBucket, + Key: awsKey, + }), + ) + .then(data => { + lastModified = data.LastModified; + next(null, data); + }) + .catch(err => next(err)), + next => + makeRequest( + { + authCredentials: backbeatAuthCredentials, + hostname: ipAddress, + port: 8000, + method: 'POST', + path: `/_/backbeat/batchdelete/${awsBucket}/${awsKey}`, + headers: { + 'if-unmodified-since': lastModified, + 'x-scal-storage-class': awsLocation, + 'x-scal-tags': JSON.stringify({ + 'scal-delete-marker': 'true', + 'scal-delete-service': 'lifecycle-transition', + }), + }, + requestBody: JSON.stringify({ + Locations: [ + { + key: awsKey, + dataStoreName: awsLocation, + }, + ], + }), + jsonResponse: true, + }, + next, + ), + next => + awsClient + .send( + new GetObjectTaggingCommand({ + Bucket: awsBucket, + Key: awsKey, + }), + ) + .then(data => { + assert.strictEqual(data.TagSet.length, 2); + data.TagSet.forEach(tag => { + const { Key, Value } = tag; + const isValidTag = + Key === 'scal-delete-marker' || Key === 'scal-delete-service'; + assert(isValidTag); + if (Key === 'scal-delete-marker') { + assert.strictEqual(Value, 'true'); + } + if (Key === 'scal-delete-service') { + assert.strictEqual(Value, 'lifecycle-transition'); + } + }); + next(null, data); + }) + .catch(err => { + assert.ifError(err); + next(err); + }), + ], + done, + ); + }, + ); - itIfLocationAzure('should delete the object if the source is Azure and ' + - 'if-unmodified-since condition is met', done => { - const blob = uuidv4(); - let lastModified; - async.series([ - next => - azureClient.getContainerClient(containerName).uploadBlockBlob(blob, 'a', 1) - .then(() => next(), next), - next => - azureClient.getContainerClient(containerName).getBlobClient(blob).getProperties() - .then(result => { - lastModified = result.lastModified; - return next(); - }, next), - next => - makeRequest({ - authCredentials: backbeatAuthCredentials, - hostname: ipAddress, - port: 8000, - method: 'POST', - path: - `/_/backbeat/batchdelete/${containerName}/${blob}`, - headers: { - 'if-unmodified-since': lastModified, - 'x-scal-storage-class': azureLocation, - 'x-scal-tags': JSON.stringify({ - 'scal-delete-marker': 'true', - 'scal-delete-service': 'lifecycle-transition', - }), - }, - requestBody: JSON.stringify({ - Locations: [{ - key: blob, - dataStoreName: azureLocation, - }], - }), - jsonResponse: true, - }, next), - next => - azureClient.getContainerClient(containerName).getBlobClient(blob).getProperties() - .then(() => assert.fail('Expected error'), err => { - assert.strictEqual(err.statusCode, 404); - return next(); - }), - ], done); - }); + itIfLocationAzure( + 'should not delete the object if the source is Azure and ' + 'if-unmodified-since condition is not met', + done => { + const blob = uuidv4(); + async.series( + [ + next => + azureClient + .getContainerClient(containerName) + .uploadBlockBlob(blob, 'a', 1) + .then(() => next(), next), + next => + makeRequest( + { + authCredentials: backbeatAuthCredentials, + hostname: ipAddress, + port: 8000, + + method: 'POST', + path: `/_/backbeat/batchdelete/${containerName}/${blob}`, + headers: { + 'if-unmodified-since': 'Sun, 31 Mar 2019 00:00:00 GMT', + 'x-scal-storage-class': azureLocation, + 'x-scal-tags': JSON.stringify({ + 'scal-delete-marker': 'true', + 'scal-delete-service': 'lifecycle-transition', + }), + }, + requestBody: JSON.stringify({ + Locations: [ + { + key: blob, + dataStoreName: azureLocation, + }, + ], + }), + jsonResponse: true, + }, + err => { + if (err && err.statusCode === 412) { + return next(); + } + return next(err); + }, + ), + next => + azureClient + .getContainerClient(containerName) + .getBlobClient(blob) + .getProperties() + .then(result => { + assert(result); + return next(); + }, next), + ], + done, + ); + }, + ); + + itIfLocationAzure( + 'should delete the object if the source is Azure and ' + 'if-unmodified-since condition is met', + done => { + const blob = uuidv4(); + let lastModified; + async.series( + [ + next => + azureClient + .getContainerClient(containerName) + .uploadBlockBlob(blob, 'a', 1) + .then(() => next(), next), + next => + azureClient + .getContainerClient(containerName) + .getBlobClient(blob) + .getProperties() + .then(result => { + lastModified = result.lastModified; + return next(); + }, next), + next => + makeRequest( + { + authCredentials: backbeatAuthCredentials, + hostname: ipAddress, + port: 8000, + method: 'POST', + path: `/_/backbeat/batchdelete/${containerName}/${blob}`, + headers: { + 'if-unmodified-since': lastModified, + 'x-scal-storage-class': azureLocation, + 'x-scal-tags': JSON.stringify({ + 'scal-delete-marker': 'true', + 'scal-delete-service': 'lifecycle-transition', + }), + }, + requestBody: JSON.stringify({ + Locations: [ + { + key: blob, + dataStoreName: azureLocation, + }, + ], + }), + jsonResponse: true, + }, + next, + ), + next => + azureClient + .getContainerClient(containerName) + .getBlobClient(blob) + .getProperties() + .then( + () => assert.fail('Expected error'), + err => { + assert.strictEqual(err.statusCode, 404); + return next(); + }, + ), + ], + done, + ); + }, + ); }); }); diff --git a/tests/sur/quota.js b/tests/sur/quota.js index c221b8512e..1543e1cc23 100644 --- a/tests/sur/quota.js +++ b/tests/sur/quota.js @@ -43,51 +43,62 @@ function createBucket(bucket, locked, cb) { if (locked) { config.ObjectLockEnabledForBucket = true; } - return s3Client.send(new CreateBucketCommand(config)) + return s3Client + .send(new CreateBucketCommand(config)) .then(data => cb(null, data)) .catch(cb); } function configureBucketVersioning(bucket, cb) { - return s3Client.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { - Status: 'Enabled', - }, - })) + return s3Client + .send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { + Status: 'Enabled', + }, + }), + ) .then(data => cb(null, data)) .catch(cb); } function putObjectLockConfiguration(bucket, cb) { - return s3Client.send(new PutObjectLockConfigurationCommand({ - Bucket: bucket, - ObjectLockConfiguration: { - ObjectLockEnabled: 'Enabled', - Rule: { - DefaultRetention: { - Mode: 'GOVERNANCE', - Days: 1, + return s3Client + .send( + new PutObjectLockConfigurationCommand({ + Bucket: bucket, + ObjectLockConfiguration: { + ObjectLockEnabled: 'Enabled', + Rule: { + DefaultRetention: { + Mode: 'GOVERNANCE', + Days: 1, + }, + }, }, - }, - }, - })) + }), + ) .then(data => cb(null, data)) .catch(cb); } function deleteBucket(bucket, cb) { - return s3Client.send(new DeleteBucketCommand({ Bucket: bucket })) + return s3Client + .send(new DeleteBucketCommand({ Bucket: bucket })) .then(data => cb(null, data)) .catch(cb); } function putObject(bucket, key, size, cb) { - return s3Client.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: Buffer.alloc(size), - })) + return s3Client + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: Buffer.alloc(size), + }), + ) .then(data => { if (!s3Config.isQuotaInflightEnabled()) { mockScuba.incrementBytesForBucket(bucket, size); @@ -111,10 +122,11 @@ function putObjectWithCustomHeader(bucket, key, size, vID, cb) { args.request.headers['x-scal-s3-version-id'] = vID; return next(args); }, - { step: 'build' } + { step: 'build' }, ); - return s3Client.send(command) + return s3Client + .send(command) .then(data => { if (!s3Config.isQuotaInflightEnabled()) { mockScuba.incrementBytesForBucket(bucket, 0); @@ -125,11 +137,14 @@ function putObjectWithCustomHeader(bucket, key, size, vID, cb) { } function copyObject(bucket, key, sourceSize, cb) { - return s3Client.send(new CopyObjectCommand({ - Bucket: bucket, - CopySource: `${bucket}/${key}`, - Key: `${key}-copy`, - })) + return s3Client + .send( + new CopyObjectCommand({ + Bucket: bucket, + CopySource: `${bucket}/${key}`, + Key: `${key}-copy`, + }), + ) .then(data => { if (!s3Config.isQuotaInflightEnabled()) { mockScuba.incrementBytesForBucket(bucket, sourceSize); @@ -140,10 +155,13 @@ function copyObject(bucket, key, sourceSize, cb) { } function deleteObject(bucket, key, size, cb) { - return s3Client.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: key, - })) + return s3Client + .send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: key, + }), + ) .then(() => { if (!s3Config.isQuotaInflightEnabled()) { mockScuba.incrementBytesForBucket(bucket, -size); @@ -154,11 +172,14 @@ function deleteObject(bucket, key, size, cb) { } function deleteVersionID(bucket, key, versionId, size, cb) { - return s3Client.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: versionId, - })) + return s3Client + .send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: versionId, + }), + ) .then(data => { if (!s3Config.isQuotaInflightEnabled()) { mockScuba.incrementBytesForBucket(bucket, -size); @@ -176,62 +197,77 @@ function objectMPU(bucket, key, parts, partSize, callback) { Bucket: bucket, Key: key, }; - return async.waterfall([ - next => s3Client.send(new CreateMultipartUploadCommand(initiateMPUParams)) - .then(data => { - uploadId = data.UploadId; - return next(); - }) - .catch(next), - next => - async.mapLimit(partNumbers, 1, (partNumber, callback) => { - const uploadPartParams = { + return async.waterfall( + [ + next => + s3Client + .send(new CreateMultipartUploadCommand(initiateMPUParams)) + .then(data => { + uploadId = data.UploadId; + return next(); + }) + .catch(next), + next => + async.mapLimit( + partNumbers, + 1, + (partNumber, callback) => { + const uploadPartParams = { + Bucket: bucket, + Key: key, + PartNumber: partNumber + 1, + UploadId: uploadId, + Body: Buffer.alloc(partSize), + }; + return s3Client + .send(new UploadPartCommand(uploadPartParams)) + .then(data => callback(null, data.ETag)) + .catch(callback); + }, + (err, results) => { + if (err) { + return next(err); + } + ETags = results; + return next(); + }, + ), + next => { + const params = { Bucket: bucket, Key: key, - PartNumber: partNumber + 1, + MultipartUpload: { + Parts: partNumbers.map(n => ({ + ETag: ETags[n], + PartNumber: n + 1, + })), + }, UploadId: uploadId, - Body: Buffer.alloc(partSize), }; - return s3Client.send(new UploadPartCommand(uploadPartParams)) - .then(data => callback(null, data.ETag)) - .catch(callback); - }, (err, results) => { - if (err) { - return next(err); - } - ETags = results; - return next(); - }), - next => { - const params = { - Bucket: bucket, - Key: key, - MultipartUpload: { - Parts: partNumbers.map(n => ({ - ETag: ETags[n], - PartNumber: n + 1, - })), - }, - UploadId: uploadId, - }; - return s3Client.send(new CompleteMultipartUploadCommand(params)) - .then(data => next(null, data)) - .catch(next); + return s3Client + .send(new CompleteMultipartUploadCommand(params)) + .then(data => next(null, data)) + .catch(next); + }, + ], + err => { + if (!err && !s3Config.isQuotaInflightEnabled()) { + mockScuba.incrementBytesForBucket(bucket, parts * partSize); + } + return callback(err, uploadId); }, - ], err => { - if (!err && !s3Config.isQuotaInflightEnabled()) { - mockScuba.incrementBytesForBucket(bucket, parts * partSize); - } - return callback(err, uploadId); - }); + ); } function abortMPU(bucket, key, uploadId, size, callback) { - return s3Client.send(new AbortMultipartUploadCommand({ - Bucket: bucket, - Key: key, - UploadId: uploadId, - })) + return s3Client + .send( + new AbortMultipartUploadCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + }), + ) .then(data => { if (!s3Config.isQuotaInflightEnabled()) { mockScuba.incrementBytesForBucket(bucket, -size); @@ -253,89 +289,101 @@ function uploadPartCopy(bucket, key, partNumber, partSize, sleepDuration, keyToC if (!s3Config.isQuotaInflightEnabled()) { mockScuba.incrementBytesForBucket(bucket, parts * partSize); } - return async.waterfall([ - next => s3Client.send(new CreateMultipartUploadCommand(initiateMPUParams)) - .then(data => { - uploadId = data.UploadId; - return next(); - }) - .catch(next), - next => { - const uploadPartParams = { - Bucket: bucket, - Key: key, - PartNumber: partNumber + 1, - UploadId: uploadId, - Body: Buffer.alloc(partSize), - }; - return s3Client.send(new UploadPartCommand(uploadPartParams)) - .then(data => { - ETags[partNumber] = data.ETag; - return next(); - }) - .catch(next); - }, - next => wait(sleepDuration, next), - next => { - const copyPartParams = { - Bucket: bucket, - CopySource: `${bucket}/${keyToCopy}`, - Key: `${key}-copy`, - PartNumber: partNumber + 1, - UploadId: uploadId, - }; - return s3Client.send(new UploadPartCopyCommand(copyPartParams)) - .then(data => { - ETags[partNumber] = data.CopyPartResult.ETag; - return next(null, data.CopyPartResult.ETag); - }) - .catch(next); - }, - next => { - const params = { - Bucket: bucket, - Key: key, - MultipartUpload: { - Parts: partNumbers.map(n => ({ - ETag: ETags[n], - PartNumber: n + 1, - })), - }, - UploadId: uploadId, - }; - return s3Client.send(new CompleteMultipartUploadCommand(params)) - .then(() => next()) - .catch(next); + return async.waterfall( + [ + next => + s3Client + .send(new CreateMultipartUploadCommand(initiateMPUParams)) + .then(data => { + uploadId = data.UploadId; + return next(); + }) + .catch(next), + next => { + const uploadPartParams = { + Bucket: bucket, + Key: key, + PartNumber: partNumber + 1, + UploadId: uploadId, + Body: Buffer.alloc(partSize), + }; + return s3Client + .send(new UploadPartCommand(uploadPartParams)) + .then(data => { + ETags[partNumber] = data.ETag; + return next(); + }) + .catch(next); + }, + next => wait(sleepDuration, next), + next => { + const copyPartParams = { + Bucket: bucket, + CopySource: `${bucket}/${keyToCopy}`, + Key: `${key}-copy`, + PartNumber: partNumber + 1, + UploadId: uploadId, + }; + return s3Client + .send(new UploadPartCopyCommand(copyPartParams)) + .then(data => { + ETags[partNumber] = data.CopyPartResult.ETag; + return next(null, data.CopyPartResult.ETag); + }) + .catch(next); + }, + next => { + const params = { + Bucket: bucket, + Key: key, + MultipartUpload: { + Parts: partNumbers.map(n => ({ + ETag: ETags[n], + PartNumber: n + 1, + })), + }, + UploadId: uploadId, + }; + return s3Client + .send(new CompleteMultipartUploadCommand(params)) + .then(() => next()) + .catch(next); + }, + ], + err => { + if (err && !s3Config.isQuotaInflightEnabled()) { + mockScuba.incrementBytesForBucket(bucket, -(parts * partSize)); + } + return callback(err, uploadId); }, - ], err => { - if (err && !s3Config.isQuotaInflightEnabled()) { - mockScuba.incrementBytesForBucket(bucket, -(parts * partSize)); - } - return callback(err, uploadId); - }); + ); } function restoreObject(bucket, key, size, callback) { - return s3Client.send(new RestoreObjectCommand({ - Bucket: bucket, - Key: key, - RestoreRequest: { - Days: 1, - }, - })).then(data => { - if (!s3Config.isQuotaInflightEnabled()) { - mockScuba.incrementBytesForBucket(bucket, size); - } - return callback(null, data); - }) - .catch(callback); + return s3Client + .send( + new RestoreObjectCommand({ + Bucket: bucket, + Key: key, + RestoreRequest: { + Days: 1, + }, + }), + ) + .then(data => { + if (!s3Config.isQuotaInflightEnabled()) { + mockScuba.incrementBytesForBucket(bucket, size); + } + return callback(null, data); + }) + .catch(callback); } function multiObjectDelete(bucket, keys, size, callback) { if (!s3Config.isQuotaInflightEnabled()) { mockScuba.incrementBytesForBucket(bucket, -size); } - const deleteObjectsParams = keys.map(key => ({ Key: key })); + const deleteObjectsParams = keys.map(key => ({ Key: key })); const command = new DeleteObjectsCommand({ Bucket: bucket, Delete: { @@ -343,8 +391,9 @@ function multiObjectDelete(bucket, keys, size, callback) { Quiet: false, }, }); - - return s3Client.send(command) + + return s3Client + .send(command) .then(data => { callback(null, data); }) @@ -356,53 +405,55 @@ function multiObjectDelete(bucket, keys, size, callback) { }); } -(process.env.S3METADATA === 'mongodb' ? describe : describe.skip)('quota evaluation with scuba metrics', - function t() { - this.timeout(30000); - const scuba = new MockScuba(); - const putQuotaVerb = 'PUT'; - const config = { - accessKey: memCredentials.default.accessKey, - secretKey: memCredentials.default.secretKey, - }; - mockScuba = scuba; - - before(done => { - const config = getConfig('default', { - maxRetries: 0, - }); - - s3Client = new S3Client({ - ...config, - // Disable ALL automatic checksum handling - requestChecksumCalculation: 'WHEN_REQUIRED', - responseChecksumValidation: 'WHEN_REQUIRED', - checksumDisabled: true, - disableRequestCompression: true, - // Force the client to not add automatic headers - useGlobalEndpoint: false, - }); +(process.env.S3METADATA === 'mongodb' ? describe : describe.skip)('quota evaluation with scuba metrics', function t() { + this.timeout(30000); + const scuba = new MockScuba(); + const putQuotaVerb = 'PUT'; + const config = { + accessKey: memCredentials.default.accessKey, + secretKey: memCredentials.default.secretKey, + }; + mockScuba = scuba; - scuba.start(); - metadata.setup(err => wait(2000, () => done(err))); + before(done => { + const config = getConfig('default', { + maxRetries: 0, }); - afterEach(() => { - scuba.reset(); + s3Client = new S3Client({ + ...config, + // Disable ALL automatic checksum handling + requestChecksumCalculation: 'WHEN_REQUIRED', + responseChecksumValidation: 'WHEN_REQUIRED', + checksumDisabled: true, + disableRequestCompression: true, + // Force the client to not add automatic headers + useGlobalEndpoint: false, }); - after(() => { - scuba.stop(); - }); + scuba.start(); + metadata.setup(err => wait(2000, () => done(err))); + }); + + afterEach(() => { + scuba.reset(); + }); + + after(() => { + scuba.stop(); + }); - it('should return QuotaExceeded when trying to PutObject in a bucket with quota', done => { - const bucket = 'quota-test-bucket1'; - const key = 'quota-test-object'; - const size = 1024; - return async.series([ + it('should return QuotaExceeded when trying to PutObject in a bucket with quota', done => { + const bucket = 'quota-test-bucket1'; + const key = 'quota-test-object'; + const size = 1024; + return async.series( + [ next => createBucket(bucket, false, next), - next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, - JSON.stringify(quota), config).then(() => next()).catch(err => next(err)), + next => + sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, JSON.stringify(quota), config) + .then(() => next()) + .catch(err => next(err)), next => { putObject(bucket, key, size, err => { try { @@ -414,80 +465,99 @@ function multiObjectDelete(bucket, keys, size, callback) { }); }, next => deleteBucket(bucket, next), - ], done); - }); + ], + done, + ); + }); - it('should return QuotaExceeded when trying to copyObject in a versioned bucket with quota', done => { - const bucket = 'quota-test-bucket12'; - const key = 'quota-test-object'; - const size = 900; - let vID = null; - return async.series([ + it('should return QuotaExceeded when trying to copyObject in a versioned bucket with quota', done => { + const bucket = 'quota-test-bucket12'; + const key = 'quota-test-object'; + const size = 900; + let vID = null; + return async.series( + [ next => createBucket(bucket, false, next), next => configureBucketVersioning(bucket, next), - next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, - JSON.stringify(quota), config).then(() => next()).catch(err => next(err)), - next => putObject(bucket, key, size, (err, data) => { - assert.ifError(err); - vID = data.VersionId; - return next(); - }), - next => wait(inflightFlushFrequencyMS * 2, next), - next => copyObject(bucket, key, size, err => { - try { - assert.strictEqual(err.name, 'QuotaExceeded'); + next => + sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, JSON.stringify(quota), config) + .then(() => next()) + .catch(err => next(err)), + next => + putObject(bucket, key, size, (err, data) => { + assert.ifError(err); + vID = data.VersionId; return next(); - } catch (assertError) { - return next(assertError); - } - }), + }), + next => wait(inflightFlushFrequencyMS * 2, next), + next => + copyObject(bucket, key, size, err => { + try { + assert.strictEqual(err.name, 'QuotaExceeded'); + return next(); + } catch (assertError) { + return next(assertError); + } + }), next => deleteVersionID(bucket, key, vID, size, next), next => deleteBucket(bucket, next), - ], done); - }); + ], + done, + ); + }); - it('should return QuotaExceeded when trying to CopyObject in a bucket with quota', done => { - const bucket = 'quota-test-bucket2'; - const key = 'quota-test-object'; - const size = 900; - return async.series([ + it('should return QuotaExceeded when trying to CopyObject in a bucket with quota', done => { + const bucket = 'quota-test-bucket2'; + const key = 'quota-test-object'; + const size = 900; + return async.series( + [ next => createBucket(bucket, false, next), - next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, - JSON.stringify(quota), config).then(() => next()).catch(err => next(err)), + next => + sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, JSON.stringify(quota), config) + .then(() => next()) + .catch(err => next(err)), next => putObject(bucket, key, size, next), next => wait(inflightFlushFrequencyMS * 2, next), - next => copyObject(bucket, key, size, err => { - try { - assert.strictEqual(err.name, 'QuotaExceeded'); - return next(); - } catch (assertError) { - return next(assertError); - } - }), + next => + copyObject(bucket, key, size, err => { + try { + assert.strictEqual(err.name, 'QuotaExceeded'); + return next(); + } catch (assertError) { + return next(assertError); + } + }), next => deleteObject(bucket, key, size, next), next => deleteBucket(bucket, next), - ], done); - }); + ], + done, + ); + }); - it('should return QuotaExceeded when trying to complete MPU in a bucket with quota', done => { - const bucket = 'quota-test-bucket3'; - const key = 'quota-test-object'; - const parts = 5; - const partSize = 1024 * 1024 * 6; - let uploadId = null; - return async.series([ + it('should return QuotaExceeded when trying to complete MPU in a bucket with quota', done => { + const bucket = 'quota-test-bucket3'; + const key = 'quota-test-object'; + const parts = 5; + const partSize = 1024 * 1024 * 6; + let uploadId = null; + return async.series( + [ next => createBucket(bucket, false, next), - next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, - JSON.stringify(quota), config).then(() => next()).catch(err => next(err)), - next => objectMPU(bucket, key, parts, partSize, (err, _uploadId) => { - uploadId = _uploadId; - try { - assert.strictEqual(err.name, 'QuotaExceeded'); - return next(); - } catch (assertError) { - return next(assertError); - } - }), + next => + sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, JSON.stringify(quota), config) + .then(() => next()) + .catch(err => next(err)), + next => + objectMPU(bucket, key, parts, partSize, (err, _uploadId) => { + uploadId = _uploadId; + try { + assert.strictEqual(err.name, 'QuotaExceeded'); + return next(); + } catch (assertError) { + return next(assertError); + } + }), next => abortMPU(bucket, key, uploadId, 0, next), next => wait(inflightFlushFrequencyMS * 2, next), next => { @@ -495,130 +565,187 @@ function multiObjectDelete(bucket, keys, size, callback) { return next(); }, next => deleteBucket(bucket, next), - ], done); - }); + ], + done, + ); + }); - it('should not return QuotaExceeded if the quota is not exceeded', done => { - const bucket = 'quota-test-bucket4'; - const key = 'quota-test-object'; - const size = 300; - return async.series([ + it('should not return QuotaExceeded if the quota is not exceeded', done => { + const bucket = 'quota-test-bucket4'; + const key = 'quota-test-object'; + const size = 300; + return async.series( + [ next => createBucket(bucket, false, next), - next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, - JSON.stringify(quota), config).then(() => next()).catch(err => next(err)), - next => putObject(bucket, key, size, err => { - assert.ifError(err); - return next(); - }), + next => + sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, JSON.stringify(quota), config) + .then(() => next()) + .catch(err => next(err)), + next => + putObject(bucket, key, size, err => { + assert.ifError(err); + return next(); + }), next => deleteObject(bucket, key, size, next), next => deleteBucket(bucket, next), - ], done); - }); + ], + done, + ); + }); - it('should not evaluate quotas if the backend is not available', done => { - scuba.stop(); - const bucket = 'quota-test-bucket5'; - const key = 'quota-test-object'; - const size = 1024; - return async.series([ + it('should not evaluate quotas if the backend is not available', done => { + scuba.stop(); + const bucket = 'quota-test-bucket5'; + const key = 'quota-test-object'; + const size = 1024; + return async.series( + [ next => createBucket(bucket, false, next), - next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, - JSON.stringify(quota), config).then(() => next()).catch(err => next(err)), - next => putObject(bucket, key, size, err => { - assert.ifError(err); - return next(); - }), + next => + sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, JSON.stringify(quota), config) + .then(() => next()) + .catch(err => next(err)), + next => + putObject(bucket, key, size, err => { + assert.ifError(err); + return next(); + }), next => deleteObject(bucket, key, size, next), next => deleteBucket(bucket, next), - ], err => { + ], + err => { assert.ifError(err); scuba.start(); return wait(2000, done); - }); - }); + }, + ); + }); - it('should return QuotaExceeded when trying to copy a part in a bucket with quota', done => { - const bucket = 'quota-test-bucket6'; - const key = 'quota-test-object-copy'; - const keyToCopy = 'quota-test-existing'; - const parts = 5; - const partSize = 1024 * 1024 * 6; - let uploadId = null; - return async.series([ + it('should return QuotaExceeded when trying to copy a part in a bucket with quota', done => { + const bucket = 'quota-test-bucket6'; + const key = 'quota-test-object-copy'; + const keyToCopy = 'quota-test-existing'; + const parts = 5; + const partSize = 1024 * 1024 * 6; + let uploadId = null; + return async.series( + [ next => createBucket(bucket, false, next), - next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, - JSON.stringify({ quota: Math.round(partSize * 2.5) }), config) - .then(() => next()).catch(err => next(err)), + next => + sendRequest( + putQuotaVerb, + '127.0.0.1:8000', + `/${bucket}/?quota=true`, + JSON.stringify({ quota: Math.round(partSize * 2.5) }), + config, + ) + .then(() => next()) + .catch(err => next(err)), next => putObject(bucket, keyToCopy, partSize, next), - next => uploadPartCopy(bucket, key, parts, partSize, inflightFlushFrequencyMS * 2, keyToCopy, - (err, _uploadId) => { - uploadId = _uploadId; - try { - assert.strictEqual(err.name, 'QuotaExceeded'); - return next(); - } catch (assertError) { - return next(assertError); - } - }), + next => + uploadPartCopy( + bucket, + key, + parts, + partSize, + inflightFlushFrequencyMS * 2, + keyToCopy, + (err, _uploadId) => { + uploadId = _uploadId; + try { + assert.strictEqual(err.name, 'QuotaExceeded'); + return next(); + } catch (assertError) { + return next(assertError); + } + }, + ), next => abortMPU(bucket, key, uploadId, parts * partSize, next), next => deleteObject(bucket, keyToCopy, partSize, next), next => deleteBucket(bucket, next), - ], done); - }); + ], + done, + ); + }); - it('should return QuotaExceeded when trying to restore an object in a bucket with quota', done => { - const bucket = 'quota-test-bucket7'; - const key = 'quota-test-object'; - const size = 900; - let vID = null; - return async.series([ + it('should return QuotaExceeded when trying to restore an object in a bucket with quota', done => { + const bucket = 'quota-test-bucket7'; + const key = 'quota-test-object'; + const size = 900; + let vID = null; + return async.series( + [ next => createBucket(bucket, false, next), next => configureBucketVersioning(bucket, next), - next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, - JSON.stringify(quota), config).then(() => next()).catch(err => next(err)), - next => putObject(bucket, key, size, (err, data) => { - assert.ifError(err); - vID = data.VersionId; - return next(); - }), - next => fakeMetadataArchive(bucket, key, vID, { - archiveInfo: {}, - }, next), - next => wait(inflightFlushFrequencyMS * 2, next), - next => restoreObject(bucket, key, size, err => { - try { - assert.strictEqual(err.name, 'QuotaExceeded'); + next => + sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, JSON.stringify(quota), config) + .then(() => next()) + .catch(err => next(err)), + next => + putObject(bucket, key, size, (err, data) => { + assert.ifError(err); + vID = data.VersionId; return next(); - } catch (assertError) { - return next(assertError); - } - }), + }), + next => + fakeMetadataArchive( + bucket, + key, + vID, + { + archiveInfo: {}, + }, + next, + ), + next => wait(inflightFlushFrequencyMS * 2, next), + next => + restoreObject(bucket, key, size, err => { + try { + assert.strictEqual(err.name, 'QuotaExceeded'); + return next(); + } catch (assertError) { + return next(assertError); + } + }), next => deleteVersionID(bucket, key, vID, size, next), next => deleteBucket(bucket, next), - ], done); - }); + ], + done, + ); + }); - it('should not update the inflights if the quota check is passing but the object is already restored', done => { - const bucket = 'quota-test-bucket14'; - const key = 'quota-test-object'; - const size = 100; - let vID = null; - return async.series([ + it('should not update the inflights if the quota check is passing but the object is already restored', done => { + const bucket = 'quota-test-bucket14'; + const key = 'quota-test-object'; + const size = 100; + let vID = null; + return async.series( + [ next => createBucket(bucket, false, next), next => configureBucketVersioning(bucket, next), - next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, - JSON.stringify(quota), config).then(() => next()).catch(err => next(err)), - next => putObject(bucket, key, size, (err, data) => { - assert.ifError(err); - vID = data.VersionId; - return next(); - }), - next => fakeMetadataArchive(bucket, key, vID, { - archiveInfo: {}, - restoreRequestedAt: new Date(0).toString(), - restoreCompletedAt: new Date(0).toString() + 1, - restoreRequestedDays: 5, - }, next), + next => + sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, JSON.stringify(quota), config) + .then(() => next()) + .catch(err => next(err)), + next => + putObject(bucket, key, size, (err, data) => { + assert.ifError(err); + vID = data.VersionId; + return next(); + }), + next => + fakeMetadataArchive( + bucket, + key, + vID, + { + archiveInfo: {}, + restoreRequestedAt: new Date(0).toString(), + restoreCompletedAt: new Date(0).toString() + 1, + restoreRequestedDays: 5, + }, + next, + ), next => wait(inflightFlushFrequencyMS * 2, next), next => { assert.strictEqual(scuba.getInflightsForBucket(bucket), size); @@ -632,34 +759,42 @@ function multiObjectDelete(bucket, keys, size, callback) { }, next => deleteVersionID(bucket, key, vID, size, next), next => deleteBucket(bucket, next), - ], done); - }); + ], + done, + ); + }); - it('should allow writes after deleting data with quotas', done => { - const bucket = 'quota-test-bucket8'; - const key = 'quota-test-object'; - const size = 400; - return async.series([ + it('should allow writes after deleting data with quotas', done => { + const bucket = 'quota-test-bucket8'; + const key = 'quota-test-object'; + const size = 400; + return async.series( + [ next => createBucket(bucket, false, next), - next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, - JSON.stringify(quota), config).then(() => next()).catch(err => next(err)), - next => putObject(bucket, `${key}1`, size, err => { - assert.ifError(err); - return next(); - }), - next => putObject(bucket, `${key}2`, size, err => { - assert.ifError(err); - return next(); - }), - next => wait(inflightFlushFrequencyMS * 2, next), - next => putObject(bucket, `${key}3`, size, err => { - try { - assert.strictEqual(err.name, 'QuotaExceeded'); + next => + sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, JSON.stringify(quota), config) + .then(() => next()) + .catch(err => next(err)), + next => + putObject(bucket, `${key}1`, size, err => { + assert.ifError(err); + return next(); + }), + next => + putObject(bucket, `${key}2`, size, err => { + assert.ifError(err); return next(); - } catch (assertError) { - return next(assertError); - } - }), + }), + next => wait(inflightFlushFrequencyMS * 2, next), + next => + putObject(bucket, `${key}3`, size, err => { + try { + assert.strictEqual(err.name, 'QuotaExceeded'); + return next(); + } catch (assertError) { + return next(assertError); + } + }), next => wait(inflightFlushFrequencyMS * 2, next), next => { assert.strictEqual(scuba.getInflightsForBucket(bucket), size * 2); @@ -668,38 +803,52 @@ function multiObjectDelete(bucket, keys, size, callback) { next => wait(inflightFlushFrequencyMS * 2, next), next => deleteObject(bucket, `${key}2`, size, next), next => wait(inflightFlushFrequencyMS * 2, next), - next => putObject(bucket, `${key}4`, size, err => { - assert.ifError(err); - return next(); - }), + next => + putObject(bucket, `${key}4`, size, err => { + assert.ifError(err); + return next(); + }), next => deleteObject(bucket, `${key}1`, size, next), next => deleteObject(bucket, `${key}3`, size, next), next => deleteObject(bucket, `${key}4`, size, next), next => deleteBucket(bucket, next), - ], done); - }); + ], + done, + ); + }); - it('should allow writes after deleting data with quotas below the current number of inflights', done => { - const bucket = 'quota-test-bucket8'; - const key = 'quota-test-object'; - const size = 400; - if (!s3Config.isQuotaInflightEnabled()) { - return done(); - } - return async.series([ + it('should allow writes after deleting data with quotas below the current number of inflights', done => { + const bucket = 'quota-test-bucket8'; + const key = 'quota-test-object'; + const size = 400; + if (!s3Config.isQuotaInflightEnabled()) { + return done(); + } + return async.series( + [ next => createBucket(bucket, false, next), // Set the quota to 10 * size (4000) - next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, - JSON.stringify({ quota: 10 * size }), config).then(() => next()).catch(err => next(err)), + next => + sendRequest( + putQuotaVerb, + '127.0.0.1:8000', + `/${bucket}/?quota=true`, + JSON.stringify({ quota: 10 * size }), + config, + ) + .then(() => next()) + .catch(err => next(err)), // Simulate previous operations since last metrics update (4000 bytes) - next => putObject(bucket, `${key}1`, 5 * size, err => { - assert.ifError(err); - return next(); - }), - next => putObject(bucket, `${key}2`, 5 * size, err => { - assert.ifError(err); - return next(); - }), + next => + putObject(bucket, `${key}1`, 5 * size, err => { + assert.ifError(err); + return next(); + }), + next => + putObject(bucket, `${key}2`, 5 * size, err => { + assert.ifError(err); + return next(); + }), next => wait(inflightFlushFrequencyMS * 2, next), // After metrics update, set the inflights to 0 (simulate end of metrics update) next => { @@ -708,14 +857,15 @@ function multiObjectDelete(bucket, keys, size, callback) { }, // Here we have 0 inflight but the stored bytes are 4000 (equal to the quota) // Should reject new write with QuotaExceeded (4000 + 400) - next => putObject(bucket, `${key}3`, size, err => { - try { - assert.strictEqual(err.name, 'QuotaExceeded'); - return next(); - } catch (assertError) { - return next(assertError); - } - }), + next => + putObject(bucket, `${key}3`, size, err => { + try { + assert.strictEqual(err.name, 'QuotaExceeded'); + return next(); + } catch (assertError) { + return next(assertError); + } + }), next => wait(inflightFlushFrequencyMS * 2, next), // Should still have 0 as inflight next => { @@ -726,37 +876,45 @@ function multiObjectDelete(bucket, keys, size, callback) { // Now delete one object (2000 bytes), it should let us write again next => deleteObject(bucket, `${key}1`, size, next), next => wait(inflightFlushFrequencyMS * 2, next), - next => putObject(bucket, `${key}4`, 5 * size, err => { - assert.ifError(err); - return next(); - }), + next => + putObject(bucket, `${key}4`, 5 * size, err => { + assert.ifError(err); + return next(); + }), // Cleanup next => deleteObject(bucket, `${key}2`, size, next), next => deleteObject(bucket, `${key}4`, size, next), next => deleteBucket(bucket, next), - ], done); - }); + ], + done, + ); + }); - it('should not increase the inflights when the object is being rewritten with a smaller object', done => { - const bucket = 'quota-test-bucket9'; - const key = 'quota-test-object'; - const size = 400; - return async.series([ + it('should not increase the inflights when the object is being rewritten with a smaller object', done => { + const bucket = 'quota-test-bucket9'; + const key = 'quota-test-object'; + const size = 400; + return async.series( + [ next => createBucket(bucket, false, next), - next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, - JSON.stringify(quota), config).then(() => next()).catch(err => next(err)), - next => putObject(bucket, key, size, err => { - assert.ifError(err); - return next(); - }), + next => + sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, JSON.stringify(quota), config) + .then(() => next()) + .catch(err => next(err)), + next => + putObject(bucket, key, size, err => { + assert.ifError(err); + return next(); + }), next => wait(inflightFlushFrequencyMS * 2, next), - next => putObject(bucket, key, size - 100, err => { - assert.ifError(err); - if (!s3Config.isQuotaInflightEnabled()) { - mockScuba.incrementBytesForBucket(bucket, -size); - } - return next(); - }), + next => + putObject(bucket, key, size - 100, err => { + assert.ifError(err); + if (!s3Config.isQuotaInflightEnabled()) { + mockScuba.incrementBytesForBucket(bucket, -size); + } + return next(); + }), next => wait(inflightFlushFrequencyMS * 2, next), next => { assert.strictEqual(scuba.getInflightsForBucket(bucket), size - 100); @@ -764,16 +922,21 @@ function multiObjectDelete(bucket, keys, size, callback) { }, next => deleteObject(bucket, key, size, next), next => deleteBucket(bucket, next), - ], done); - }); - it('should decrease the inflights when performing multi object delete', done => { - const bucket = 'quota-test-bucket10'; - const key = 'quota-test-object'; - const size = 400; - return async.series([ + ], + done, + ); + }); + it('should decrease the inflights when performing multi object delete', done => { + const bucket = 'quota-test-bucket10'; + const key = 'quota-test-object'; + const size = 400; + return async.series( + [ next => createBucket(bucket, false, next), - next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, - JSON.stringify(quota), config).then(() => next()).catch(err => next(err)), + next => + sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, JSON.stringify(quota), config) + .then(() => next()) + .catch(err => next(err)), next => { putObject(bucket, `${key}1`, size, err => { assert.ifError(err); @@ -787,11 +950,11 @@ function multiObjectDelete(bucket, keys, size, callback) { }); }, next => wait(inflightFlushFrequencyMS * 2, next), - next => + next => multiObjectDelete(bucket, [`${key}1`, `${key}2`], size * 2, err => { assert.ifError(err); return next(); - }), + }), next => wait(inflightFlushFrequencyMS * 2, next), next => { assert.strictEqual(scuba.getInflightsForBucket(bucket), 0); @@ -800,162 +963,211 @@ function multiObjectDelete(bucket, keys, size, callback) { next => { deleteBucket(bucket, next); }, - ], done); - }); + ], + done, + ); + }); - it('should allow writes after multi-deleting data with quotas below the current number of inflights', done => { - const bucket = 'quota-test-bucket10'; - const key = 'quota-test-object'; - const size = 400; - if (!s3Config.isQuotaInflightEnabled()) { - return done(); - } - return async.series([ + it('should allow writes after multi-deleting data with quotas below the current number of inflights', done => { + const bucket = 'quota-test-bucket10'; + const key = 'quota-test-object'; + const size = 400; + if (!s3Config.isQuotaInflightEnabled()) { + return done(); + } + return async.series( + [ next => createBucket(bucket, false, next), - next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, - JSON.stringify({ quota: size * 10 }), config).then(() => next()).catch(err => next(err)), - next => putObject(bucket, `${key}1`, size * 5, err => { - assert.ifError(err); - return next(); - }), - next => putObject(bucket, `${key}2`, size * 5, err => { - assert.ifError(err); - return next(); - }), + next => + sendRequest( + putQuotaVerb, + '127.0.0.1:8000', + `/${bucket}/?quota=true`, + JSON.stringify({ quota: size * 10 }), + config, + ) + .then(() => next()) + .catch(err => next(err)), + next => + putObject(bucket, `${key}1`, size * 5, err => { + assert.ifError(err); + return next(); + }), + next => + putObject(bucket, `${key}2`, size * 5, err => { + assert.ifError(err); + return next(); + }), next => wait(inflightFlushFrequencyMS * 2, next), next => { scuba.setInflightAsCapacity(bucket); return next(); }, - next => putObject(bucket, `${key}3`, size, err => { - try { - assert.strictEqual(err.name, 'QuotaExceeded'); - return next(); - } catch (assertError) { - return next(assertError); - } - }), + next => + putObject(bucket, `${key}3`, size, err => { + try { + assert.strictEqual(err.name, 'QuotaExceeded'); + return next(); + } catch (assertError) { + return next(assertError); + } + }), next => wait(inflightFlushFrequencyMS * 2, next), next => { assert.strictEqual(scuba.getInflightsForBucket(bucket), 0); return next(); }, - next => multiObjectDelete(bucket, [`${key}1`, `${key}2`], size * 10, err => { - assert.ifError(err); - return next(); - }), + next => + multiObjectDelete(bucket, [`${key}1`, `${key}2`], size * 10, err => { + assert.ifError(err); + return next(); + }), next => wait(inflightFlushFrequencyMS * 2, next), - next => putObject(bucket, `${key}4`, size * 5, err => { - assert.ifError(err); - return next(); - }), + next => + putObject(bucket, `${key}4`, size * 5, err => { + assert.ifError(err); + return next(); + }), next => deleteObject(bucket, `${key}4`, size * 5, next), next => deleteBucket(bucket, next), - ], done); - }); + ], + done, + ); + }); - it('should not update the inflights if the API errored after evaluating quotas (deletion)', done => { - const bucket = 'quota-test-bucket11'; - const key = 'quota-test-object'; - const size = 100; - let vID = null; - return async.series([ + it('should not update the inflights if the API errored after evaluating quotas (deletion)', done => { + const bucket = 'quota-test-bucket11'; + const key = 'quota-test-object'; + const size = 100; + let vID = null; + return async.series( + [ next => createBucket(bucket, true, next), next => putObjectLockConfiguration(bucket, next), - next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, - JSON.stringify(quota), config).then(() => next()).catch(err => next(err)), - next => putObject(bucket, key, size, (err, val) => { - assert.ifError(err); - vID = val.VersionId; - return next(); - }), + next => + sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, JSON.stringify(quota), config) + .then(() => next()) + .catch(err => next(err)), + next => + putObject(bucket, key, size, (err, val) => { + assert.ifError(err); + vID = val.VersionId; + return next(); + }), next => wait(inflightFlushFrequencyMS * 2, next), next => { assert.strictEqual(scuba.getInflightsForBucket(bucket), size); return next(); }, - next => deleteVersionID(bucket, key, vID, size, err => { - try { - assert.strictEqual(err.name, 'AccessDenied'); - next(); - } catch (assertError) { - next(assertError); - } - }), + next => + deleteVersionID(bucket, key, vID, size, err => { + try { + assert.strictEqual(err.name, 'AccessDenied'); + next(); + } catch (assertError) { + next(assertError); + } + }), next => wait(inflightFlushFrequencyMS * 2, next), next => { assert.strictEqual(scuba.getInflightsForBucket(bucket), size); return next(); }, - ], done); - }); + ], + done, + ); + }); - it('should only evaluate quota and not update inflights for PutObject with the x-scal-s3-version-id header', - done => { - const bucket = 'quota-test-bucket13'; - const key = 'quota-test-object'; - const size = 100; - let vID = null; - return async.series([ - next => createBucket(bucket, true, next), - next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, - JSON.stringify(quota), config).then(() => next()).catch(err => next(err)), - next => putObject(bucket, key, size, (err, val) => { + it('should only evaluate quota and not update inflights for PutObject with the x-scal-s3-version-id header', done => { + const bucket = 'quota-test-bucket13'; + const key = 'quota-test-object'; + const size = 100; + let vID = null; + return async.series( + [ + next => createBucket(bucket, true, next), + next => + sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, JSON.stringify(quota), config) + .then(() => next()) + .catch(err => next(err)), + next => + putObject(bucket, key, size, (err, val) => { assert.ifError(err); vID = val.VersionId; return next(); }), - next => wait(inflightFlushFrequencyMS * 2, next), - next => { - assert.strictEqual(scuba.getInflightsForBucket(bucket), size); - return next(); - }, - next => fakeMetadataArchive(bucket, key, vID, { - archiveInfo: {}, - restoreRequestedAt: new Date(0).toISOString(), - restoreRequestedDays: 7, - }, next), - // Simulate the real restore - next => putObjectWithCustomHeader(bucket, key, size, vID, err => { + next => wait(inflightFlushFrequencyMS * 2, next), + next => { + assert.strictEqual(scuba.getInflightsForBucket(bucket), size); + return next(); + }, + next => + fakeMetadataArchive( + bucket, + key, + vID, + { + archiveInfo: {}, + restoreRequestedAt: new Date(0).toISOString(), + restoreRequestedDays: 7, + }, + next, + ), + // Simulate the real restore + next => + putObjectWithCustomHeader(bucket, key, size, vID, err => { assert.ifError(err); return next(); }), - next => { - assert.strictEqual(scuba.getInflightsForBucket(bucket), size); - return next(); - }, - next => deleteVersionID(bucket, key, vID, size, next), - next => deleteBucket(bucket, next), - ], done); - }); + next => { + assert.strictEqual(scuba.getInflightsForBucket(bucket), size); + return next(); + }, + next => deleteVersionID(bucket, key, vID, size, next), + next => deleteBucket(bucket, next), + ], + done, + ); + }); - it('should allow a restore if the quota is full but the objet fits with its reserved storage space', - done => { - const bucket = 'quota-test-bucket15'; - const key = 'quota-test-object'; - const size = 1000; - let vID = null; - return async.series([ - next => createBucket(bucket, true, next), - next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, - JSON.stringify(quota), config).then(() => next()).catch(err => next(err)), - next => putObject(bucket, key, size, (err, val) => { + it('should allow a restore if the quota is full but the objet fits with its reserved storage space', done => { + const bucket = 'quota-test-bucket15'; + const key = 'quota-test-object'; + const size = 1000; + let vID = null; + return async.series( + [ + next => createBucket(bucket, true, next), + next => + sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, JSON.stringify(quota), config) + .then(() => next()) + .catch(err => next(err)), + next => + putObject(bucket, key, size, (err, val) => { assert.ifError(err); vID = val.VersionId; return next(); }), - next => wait(inflightFlushFrequencyMS * 2, next), - next => { - assert.strictEqual(scuba.getInflightsForBucket(bucket), size); - return next(); - }, - next => fakeMetadataArchive(bucket, key, vID, { - archiveInfo: {}, - restoreRequestedAt: new Date(0).toISOString(), - restoreRequestedDays: 7, - }, next), - // Put an object, the quota should be exceeded - next => putObject(bucket, `${key}-2`, size, err => { + next => wait(inflightFlushFrequencyMS * 2, next), + next => { + assert.strictEqual(scuba.getInflightsForBucket(bucket), size); + return next(); + }, + next => + fakeMetadataArchive( + bucket, + key, + vID, + { + archiveInfo: {}, + restoreRequestedAt: new Date(0).toISOString(), + restoreRequestedDays: 7, + }, + next, + ), + // Put an object, the quota should be exceeded + next => + putObject(bucket, `${key}-2`, size, err => { try { assert.strictEqual(err.name, 'QuotaExceeded'); return next(); @@ -963,58 +1175,78 @@ function multiObjectDelete(bucket, keys, size, callback) { return next(assertError); } }), - next => { - assert.strictEqual(scuba.getInflightsForBucket(bucket), size); - return next(); - }, - next => deleteVersionID(bucket, key, vID, size, next), - next => deleteBucket(bucket, next), - ], done); - }); + next => { + assert.strictEqual(scuba.getInflightsForBucket(bucket), size); + return next(); + }, + next => deleteVersionID(bucket, key, vID, size, next), + next => deleteBucket(bucket, next), + ], + done, + ); + }); - it('should reduce inflights when completing MPU with fewer parts than uploaded', done => { - const bucket = 'quota-test-bucket-mpu1'; - const key = 'quota-test-object'; - const parts = 3; - const partSize = 5 * 1024 * 1024; - const totalSize = parts * partSize; - const usedParts = 2; - let uploadId = null; - const ETags = []; + it('should reduce inflights when completing MPU with fewer parts than uploaded', done => { + const bucket = 'quota-test-bucket-mpu1'; + const key = 'quota-test-object'; + const parts = 3; + const partSize = 5 * 1024 * 1024; + const totalSize = parts * partSize; + const usedParts = 2; + let uploadId = null; + const ETags = []; - if (!s3Config.isQuotaInflightEnabled()) { - return done(); - } + if (!s3Config.isQuotaInflightEnabled()) { + return done(); + } - return async.series([ + return async.series( + [ next => createBucket(bucket, false, next), - next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, - JSON.stringify({ quota: totalSize * 2 }), config) - .then(() => next()).catch(err => next(err)), - next => s3Client.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: key, - })) - .then(data => { - uploadId = data.UploadId; - return next(); - }) - .catch(err => next(err)), - next => async.timesSeries(parts, (n, cb) => { - const uploadPartParams = { - Bucket: bucket, - Key: key, - PartNumber: n + 1, - UploadId: uploadId, - Body: Buffer.alloc(partSize), - }; - return s3Client.send(new UploadPartCommand(uploadPartParams)) + next => + sendRequest( + putQuotaVerb, + '127.0.0.1:8000', + `/${bucket}/?quota=true`, + JSON.stringify({ quota: totalSize * 2 }), + config, + ) + .then(() => next()) + .catch(err => next(err)), + next => + s3Client + .send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + }), + ) .then(data => { - ETags[n] = data.ETag; - return cb(); + uploadId = data.UploadId; + return next(); }) - .catch(cb); - }, next), + .catch(err => next(err)), + next => + async.timesSeries( + parts, + (n, cb) => { + const uploadPartParams = { + Bucket: bucket, + Key: key, + PartNumber: n + 1, + UploadId: uploadId, + Body: Buffer.alloc(partSize), + }; + return s3Client + .send(new UploadPartCommand(uploadPartParams)) + .then(data => { + ETags[n] = data.ETag; + return cb(); + }) + .catch(cb); + }, + next, + ), next => wait(inflightFlushFrequencyMS * 2, next), next => { // Verify all parts are counted in inflights @@ -1034,7 +1266,8 @@ function multiObjectDelete(bucket, keys, size, callback) { }, UploadId: uploadId, }; - return s3Client.send(new CompleteMultipartUploadCommand(params)) + return s3Client + .send(new CompleteMultipartUploadCommand(params)) .then(() => next()) .catch(err => next(err)); }, @@ -1047,47 +1280,67 @@ function multiObjectDelete(bucket, keys, size, callback) { }, next => deleteObject(bucket, key, usedParts * partSize, next), next => deleteBucket(bucket, next), - ], done); - }); + ], + done, + ); + }); - it('should reduce inflights when aborting MPU', done => { - const bucket = 'quota-test-bucket-mpu2'; - const key = 'quota-test-object'; - const parts = 3; - const partSize = 5 * 1024 * 1024; - const totalSize = parts * partSize; - let uploadId = null; + it('should reduce inflights when aborting MPU', done => { + const bucket = 'quota-test-bucket-mpu2'; + const key = 'quota-test-object'; + const parts = 3; + const partSize = 5 * 1024 * 1024; + const totalSize = parts * partSize; + let uploadId = null; - if (!s3Config.isQuotaInflightEnabled()) { - return done(); - } + if (!s3Config.isQuotaInflightEnabled()) { + return done(); + } - return async.series([ + return async.series( + [ next => createBucket(bucket, false, next), - next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, - JSON.stringify({ quota: totalSize * 2 }), config) - .then(() => next()).catch(err => next(err)), - next => s3Client.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: key, - })) - .then(data => { - uploadId = data.UploadId; - return next(); - }) - .catch(err => next(err)), - next => async.timesSeries(parts, (n, cb) => { - const uploadPartParams = { - Bucket: bucket, - Key: key, - PartNumber: n + 1, - UploadId: uploadId, - Body: Buffer.alloc(partSize), - }; - return s3Client.send(new UploadPartCommand(uploadPartParams)) - .then(data => cb(null, data)) - .catch(cb); - }, next), + next => + sendRequest( + putQuotaVerb, + '127.0.0.1:8000', + `/${bucket}/?quota=true`, + JSON.stringify({ quota: totalSize * 2 }), + config, + ) + .then(() => next()) + .catch(err => next(err)), + next => + s3Client + .send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + }), + ) + .then(data => { + uploadId = data.UploadId; + return next(); + }) + .catch(err => next(err)), + next => + async.timesSeries( + parts, + (n, cb) => { + const uploadPartParams = { + Bucket: bucket, + Key: key, + PartNumber: n + 1, + UploadId: uploadId, + Body: Buffer.alloc(partSize), + }; + return s3Client + .send(new UploadPartCommand(uploadPartParams)) + .then(data => cb(null, data)) + .catch(cb); + }, + next, + ), next => wait(inflightFlushFrequencyMS * 2, next), next => { // Verify all parts are counted in inflights @@ -1102,6 +1355,8 @@ function multiObjectDelete(bucket, keys, size, callback) { return next(); }, next => deleteBucket(bucket, next), - ], done); - }); + ], + done, + ); }); +}); diff --git a/tests/sur/routeVeeam.js b/tests/sur/routeVeeam.js index 0a9bf51554..6911d3a0d5 100644 --- a/tests/sur/routeVeeam.js +++ b/tests/sur/routeVeeam.js @@ -2,14 +2,10 @@ const assert = require('assert'); const crypto = require('crypto'); const async = require('async'); const { Scuba: MockScuba } = require('../utilities/mock/Scuba'); -const { - CreateBucketCommand, - DeleteBucketCommand, -} = require('@aws-sdk/client-s3'); +const { CreateBucketCommand, DeleteBucketCommand } = require('@aws-sdk/client-s3'); const { makeRequest } = require('../functional/raw-node/utils/makeRequest'); -const BucketUtility = - require('../functional/aws-node-sdk/lib/utility/bucket-util'); +const BucketUtility = require('../functional/aws-node-sdk/lib/utility/bucket-util'); const ipAddress = process.env.IP ? process.env.IP : '127.0.0.1'; @@ -33,9 +29,7 @@ const testCapacity = ` 0 \n`; -const testCapacityMd5 = crypto.createHash('md5') - .update(testCapacity, 'utf-8') - .digest('hex'); +const testCapacityMd5 = crypto.createHash('md5').update(testCapacity, 'utf-8').digest('hex'); const invalidTestCapacity = ` @@ -44,9 +38,7 @@ const invalidTestCapacity = ` 0 \n`; -const invalidTestCapacityMd5 = crypto.createHash('md5') - .update(invalidTestCapacity, 'utf-8') - .digest('hex'); +const invalidTestCapacityMd5 = crypto.createHash('md5').update(invalidTestCapacity, 'utf-8').digest('hex'); const testSystem = ` @@ -69,9 +61,7 @@ const testSystem = ` \n`; -const testSystemMd5 = crypto.createHash('md5') - .update(testSystem, 'utf-8') - .digest('hex'); +const testSystemMd5 = crypto.createHash('md5').update(testSystem, 'utf-8').digest('hex'); const invalidTestSystem = ` @@ -94,9 +84,7 @@ const invalidTestSystem = ` \n`; -const invalidTestSystemMd5 = crypto.createHash('md5') - .update(testSystem, 'utf-8') - .digest('hex'); +const invalidTestSystemMd5 = crypto.createHash('md5').update(testSystem, 'utf-8').digest('hex'); let bucketUtil; let s3; @@ -117,8 +105,7 @@ let s3; * @return {undefined} - and call callback */ function makeVeeamRequest(params, callback) { - const { method, headers, bucket, objectKey, - authCredentials, requestBody, queryObj } = params; + const { method, headers, bucket, objectKey, authCredentials, requestBody, queryObj } = params; const options = { authCredentials, hostname: ipAddress, @@ -167,8 +154,7 @@ function makeVeeamRequest(params, callback) { describe('veeam PUT routes:', () => { before(done => { - bucketUtil = new BucketUtility( - 'default', { signatureVersion: 'v4' }); + bucketUtil = new BucketUtility('default', { signatureVersion: 'v4' }); s3 = bucketUtil.s3; s3.send(new CreateBucketCommand({ Bucket: TEST_BUCKET })) .then(() => done()) @@ -178,7 +164,8 @@ function makeVeeamRequest(params, callback) { }); }); after(done => { - bucketUtil.empty(TEST_BUCKET) + bucketUtil + .empty(TEST_BUCKET) .then(() => s3.send(new DeleteBucketCommand({ Bucket: TEST_BUCKET }))) .then(() => done()) .catch(done); @@ -188,75 +175,85 @@ function makeVeeamRequest(params, callback) { ['.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/system.xml', testSystem, testSystemMd5], ['.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/capacity.xml', testCapacity, testCapacityMd5], ].forEach(key => { - it(`PUT ${key[0]}`, done => makeVeeamRequest({ - method: 'PUT', - bucket: TEST_BUCKET, - objectKey: key[0], - headers: { - 'content-length': key[1].length, - 'content-md5': key[2], - 'x-scal-canonical-id': testArn, - }, - authCredentials: veeamAuthCredentials, - requestBody: key[1], - }, (err, response) => { - if (err) { - // Return the error, if any - return done(err); - } - assert.strictEqual(response.statusCode, 200); - return done(); - })); + it(`PUT ${key[0]}`, done => + makeVeeamRequest( + { + method: 'PUT', + bucket: TEST_BUCKET, + objectKey: key[0], + headers: { + 'content-length': key[1].length, + 'content-md5': key[2], + 'x-scal-canonical-id': testArn, + }, + authCredentials: veeamAuthCredentials, + requestBody: key[1], + }, + (err, response) => { + if (err) { + // Return the error, if any + return done(err); + } + assert.strictEqual(response.statusCode, 200); + return done(); + }, + )); }); [ ['.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/system.xml', invalidTestSystem, invalidTestSystemMd5], ['.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/capacity.xml', invalidTestCapacity, invalidTestCapacityMd5], ].forEach(key => { - it(`PUT ${key[0]} should fail for invalid XML`, done => makeVeeamRequest({ - method: 'PUT', - bucket: TEST_BUCKET, - objectKey: key[0], - headers: { - 'content-length': key[1].length + 3, - 'content-md5': key[2], - 'x-scal-canonical-id': testArn, - }, - authCredentials: veeamAuthCredentials, - requestBody: `${key[1]}gff`, - }, err => { - assert.strictEqual(err.code, 'MalformedXML'); - return done(); - })); + it(`PUT ${key[0]} should fail for invalid XML`, done => + makeVeeamRequest( + { + method: 'PUT', + bucket: TEST_BUCKET, + objectKey: key[0], + headers: { + 'content-length': key[1].length + 3, + 'content-md5': key[2], + 'x-scal-canonical-id': testArn, + }, + authCredentials: veeamAuthCredentials, + requestBody: `${key[1]}gff`, + }, + err => { + assert.strictEqual(err.code, 'MalformedXML'); + return done(); + }, + )); }); [ ['.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/system.xml', testSystem, testSystemMd5], ['.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/capacity.xml', testCapacity, testCapacityMd5], ].forEach(key => { - it(`PUT ${key[0]} should fail if invalid credentials are sent`, done => makeVeeamRequest({ - method: 'PUT', - bucket: TEST_BUCKET, - objectKey: key[0], - headers: { - 'content-length': key[1].length + 3, - 'content-md5': key[2], - 'x-scal-canonical-id': testArn, - }, - authCredentials: badVeeamAuthCredentials, - requestBody: `${key[1]}gff`, - }, err => { - assert.strictEqual(err.code, 'InvalidAccessKeyId'); - return done(); - })); + it(`PUT ${key[0]} should fail if invalid credentials are sent`, done => + makeVeeamRequest( + { + method: 'PUT', + bucket: TEST_BUCKET, + objectKey: key[0], + headers: { + 'content-length': key[1].length + 3, + 'content-md5': key[2], + 'x-scal-canonical-id': testArn, + }, + authCredentials: badVeeamAuthCredentials, + requestBody: `${key[1]}gff`, + }, + err => { + assert.strictEqual(err.code, 'InvalidAccessKeyId'); + return done(); + }, + )); }); }); - describe('veeam GET routes:', () => { beforeEach(done => { - bucketUtil = new BucketUtility( - 'default', { signatureVersion: 'v4' }); + bucketUtil = new BucketUtility('default', { signatureVersion: 'v4' }); s3 = bucketUtil.s3; s3.send(new CreateBucketCommand({ Bucket: TEST_BUCKET })) .then(() => done()) @@ -266,7 +263,8 @@ function makeVeeamRequest(params, callback) { }); }); afterEach(done => { - bucketUtil.empty(TEST_BUCKET) + bucketUtil + .empty(TEST_BUCKET) .then(() => s3.send(new DeleteBucketCommand({ Bucket: TEST_BUCKET }))) .then(() => done()) .catch(done); @@ -278,45 +276,56 @@ function makeVeeamRequest(params, callback) { ].forEach(key => { it(`GET ${key[0]} should return the expected XML file`, done => { scuba.incrementBytesForBucket(TEST_BUCKET, 0); - async.waterfall([ - next => makeVeeamRequest({ - method: 'PUT', - bucket: TEST_BUCKET, - objectKey: key[0], - headers: { - 'content-length': key[1].length, - 'content-md5': key[2], - 'x-scal-canonical-id': testArn, - }, - authCredentials: veeamAuthCredentials, - requestBody: key[1], - }, (err, response) => { - if (err) { - return done(err); - } - assert.strictEqual(response.statusCode, 200); - return next(); - }), - next => makeVeeamRequest({ - method: 'GET', - bucket: TEST_BUCKET, - objectKey: key[0], - headers: { - 'x-scal-canonical-id': testArn, - }, - authCredentials: veeamAuthCredentials, - }, (err, response) => { - if (err) { - return done(err); - } - assert.strictEqual(response.statusCode, 200); - assert.strictEqual(response.body.replaceAll(' ', ''), key[1].replaceAll(' ', '')); - return next(); - }), - ], err => { - assert.ifError(err); - return done(); - }); + async.waterfall( + [ + next => + makeVeeamRequest( + { + method: 'PUT', + bucket: TEST_BUCKET, + objectKey: key[0], + headers: { + 'content-length': key[1].length, + 'content-md5': key[2], + 'x-scal-canonical-id': testArn, + }, + authCredentials: veeamAuthCredentials, + requestBody: key[1], + }, + (err, response) => { + if (err) { + return done(err); + } + assert.strictEqual(response.statusCode, 200); + return next(); + }, + ), + next => + makeVeeamRequest( + { + method: 'GET', + bucket: TEST_BUCKET, + objectKey: key[0], + headers: { + 'x-scal-canonical-id': testArn, + }, + authCredentials: veeamAuthCredentials, + }, + (err, response) => { + if (err) { + return done(err); + } + assert.strictEqual(response.statusCode, 200); + assert.strictEqual(response.body.replaceAll(' ', ''), key[1].replaceAll(' ', '')); + return next(); + }, + ), + ], + err => { + assert.ifError(err); + return done(); + }, + ); }); }); @@ -325,179 +334,221 @@ function makeVeeamRequest(params, callback) { ['.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/capacity.xml', testCapacity, testCapacityMd5], ].forEach(key => { it(`GET ${key[0]} should return the expected XML file for cors requests`, done => { - async.waterfall([ - next => makeVeeamRequest({ - method: 'PUT', - bucket: TEST_BUCKET, - objectKey: key[0], - headers: { - 'origin': 'http://localhost:8000', - 'content-length': key[1].length, - 'content-md5': key[2], - 'x-scal-canonical-id': testArn, - }, - authCredentials: veeamAuthCredentials, - requestBody: key[1], - }, (err, response) => { - if (err) { - return done(err); - } - assert.strictEqual(response.statusCode, 200); - return next(); - }), - next => makeVeeamRequest({ - method: 'GET', - bucket: TEST_BUCKET, - objectKey: key[0], - headers: { - 'origin': 'http://localhost:8000', - 'x-scal-canonical-id': testArn, - }, - authCredentials: veeamAuthCredentials, - }, (err, response) => { - if (err) { - return done(err); - } - assert.strictEqual(response.statusCode, 200); - assert.strictEqual(response.body.replaceAll(' ', ''), key[1].replaceAll(' ', '')); - return next(); - }), - ], err => { - assert.ifError(err); - return done(); - }); + async.waterfall( + [ + next => + makeVeeamRequest( + { + method: 'PUT', + bucket: TEST_BUCKET, + objectKey: key[0], + headers: { + origin: 'http://localhost:8000', + 'content-length': key[1].length, + 'content-md5': key[2], + 'x-scal-canonical-id': testArn, + }, + authCredentials: veeamAuthCredentials, + requestBody: key[1], + }, + (err, response) => { + if (err) { + return done(err); + } + assert.strictEqual(response.statusCode, 200); + return next(); + }, + ), + next => + makeVeeamRequest( + { + method: 'GET', + bucket: TEST_BUCKET, + objectKey: key[0], + headers: { + origin: 'http://localhost:8000', + 'x-scal-canonical-id': testArn, + }, + authCredentials: veeamAuthCredentials, + }, + (err, response) => { + if (err) { + return done(err); + } + assert.strictEqual(response.statusCode, 200); + assert.strictEqual(response.body.replaceAll(' ', ''), key[1].replaceAll(' ', '')); + return next(); + }, + ), + ], + err => { + assert.ifError(err); + return done(); + }, + ); }); }); - [ ['.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/system.xml', testSystem, testSystemMd5], ['.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/capacity.xml', testCapacity, testCapacityMd5], ].forEach(key => { - it(`GET ${key[0]} should fail if no data in bucket metadata`, done => makeVeeamRequest({ - method: 'GET', - bucket: TEST_BUCKET, - objectKey: key[0], - headers: { - 'x-scal-canonical-id': testArn, - }, - authCredentials: veeamAuthCredentials, - }, err => { - assert.strictEqual(err.code, 'NoSuchKey'); - return done(); - })); + it(`GET ${key[0]} should fail if no data in bucket metadata`, done => + makeVeeamRequest( + { + method: 'GET', + bucket: TEST_BUCKET, + objectKey: key[0], + headers: { + 'x-scal-canonical-id': testArn, + }, + authCredentials: veeamAuthCredentials, + }, + err => { + assert.strictEqual(err.code, 'NoSuchKey'); + return done(); + }, + )); }); it('GET capacity.xml should return 200 when scubaclient returns 404 (post-install scenario)', done => { // This test simulates the post-install scenario where scubaclient returns 404 // because no metrics are available yet. By not calling scuba.incrementBytesForBucket, // the mock scuba server will return 404 for this bucket. - - async.waterfall([ - next => makeVeeamRequest({ - method: 'PUT', - bucket: TEST_BUCKET, - objectKey: '.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/capacity.xml', - headers: { - 'content-length': testCapacity.length, - 'content-md5': testCapacityMd5, - 'x-scal-canonical-id': testArn, - }, - authCredentials: veeamAuthCredentials, - requestBody: testCapacity, - }, (err, response) => { - if (err) { - return done(err); - } - assert.strictEqual(response.statusCode, 200); - return next(); - }), - next => makeVeeamRequest({ - method: 'GET', - bucket: TEST_BUCKET, - objectKey: '.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/capacity.xml', - headers: { - 'x-scal-canonical-id': testArn, - }, - authCredentials: veeamAuthCredentials, - }, (err, response) => { - if (err) { - return done(err); - } - // Critical assertion: for 404 from scubaclient (no metrics yet), - // should return 200 with static capacity data (Used=0) - assert.strictEqual(response.statusCode, 200, - 'should return 200 when scubaclient returns 404 (no metrics available)'); - // Should return capacity.xml with static data - assert(response.body.includes(''), - 'should return capacity.xml content'); - assert(response.body.includes('0'), - 'Used should be 0 from static bucket metadata'); - return next(); - }), - ], err => { - assert.ifError(err); - return done(); - }); + + async.waterfall( + [ + next => + makeVeeamRequest( + { + method: 'PUT', + bucket: TEST_BUCKET, + objectKey: '.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/capacity.xml', + headers: { + 'content-length': testCapacity.length, + 'content-md5': testCapacityMd5, + 'x-scal-canonical-id': testArn, + }, + authCredentials: veeamAuthCredentials, + requestBody: testCapacity, + }, + (err, response) => { + if (err) { + return done(err); + } + assert.strictEqual(response.statusCode, 200); + return next(); + }, + ), + next => + makeVeeamRequest( + { + method: 'GET', + bucket: TEST_BUCKET, + objectKey: '.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/capacity.xml', + headers: { + 'x-scal-canonical-id': testArn, + }, + authCredentials: veeamAuthCredentials, + }, + (err, response) => { + if (err) { + return done(err); + } + // Critical assertion: for 404 from scubaclient (no metrics yet), + // should return 200 with static capacity data (Used=0) + assert.strictEqual( + response.statusCode, + 200, + 'should return 200 when scubaclient returns 404 (no metrics available)', + ); + // Should return capacity.xml with static data + assert(response.body.includes(''), 'should return capacity.xml content'); + assert( + response.body.includes('0'), + 'Used should be 0 from static bucket metadata', + ); + return next(); + }, + ), + ], + err => { + assert.ifError(err); + return done(); + }, + ); }); it('GET system.xml should return 200 even when scubaclient is down', done => { // system.xml doesn't use scubaclient, so it should always work // This test stops scuba to verify system.xml is independent of utilization metrics - async.waterfall([ - next => makeVeeamRequest({ - method: 'PUT', - bucket: TEST_BUCKET, - objectKey: '.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/system.xml', - headers: { - 'content-length': testSystem.length, - 'content-md5': testSystemMd5, - 'x-scal-canonical-id': testArn, + async.waterfall( + [ + next => + makeVeeamRequest( + { + method: 'PUT', + bucket: TEST_BUCKET, + objectKey: '.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/system.xml', + headers: { + 'content-length': testSystem.length, + 'content-md5': testSystemMd5, + 'x-scal-canonical-id': testArn, + }, + authCredentials: veeamAuthCredentials, + requestBody: testSystem, + }, + (err, response) => { + if (err) { + return done(err); + } + assert.strictEqual(response.statusCode, 200); + return next(); + }, + ), + next => { + // Stop scuba - system.xml should still work + scuba.stop(); + return next(); }, - authCredentials: veeamAuthCredentials, - requestBody: testSystem, - }, (err, response) => { - if (err) { - return done(err); - } - assert.strictEqual(response.statusCode, 200); - return next(); - }), - next => { - // Stop scuba - system.xml should still work - scuba.stop(); - return next(); + next => + makeVeeamRequest( + { + method: 'GET', + bucket: TEST_BUCKET, + objectKey: '.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/system.xml', + headers: { + 'x-scal-canonical-id': testArn, + }, + authCredentials: veeamAuthCredentials, + }, + (err, response) => { + if (err) { + return done(err); + } + assert.strictEqual( + response.statusCode, + 200, + 'system.xml should always return 200 even when scuba is down', + ); + assert.strictEqual(response.body.replaceAll(' ', ''), testSystem.replaceAll(' ', '')); + return next(); + }, + ), + ], + err => { + // Restart scuba for subsequent tests + scuba.start(); + assert.ifError(err); + return done(); }, - next => makeVeeamRequest({ - method: 'GET', - bucket: TEST_BUCKET, - objectKey: '.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/system.xml', - headers: { - 'x-scal-canonical-id': testArn, - }, - authCredentials: veeamAuthCredentials, - }, (err, response) => { - if (err) { - return done(err); - } - assert.strictEqual(response.statusCode, 200, - 'system.xml should always return 200 even when scuba is down'); - assert.strictEqual(response.body.replaceAll(' ', ''), testSystem.replaceAll(' ', '')); - return next(); - }), - ], err => { - // Restart scuba for subsequent tests - scuba.start(); - assert.ifError(err); - return done(); - }); + ); }); }); describe('veeam DELETE routes:', () => { beforeEach(done => { - bucketUtil = new BucketUtility( - 'default', { signatureVersion: 'v4' }); + bucketUtil = new BucketUtility('default', { signatureVersion: 'v4' }); s3 = bucketUtil.s3; s3.send(new CreateBucketCommand({ Bucket: TEST_BUCKET })) .then(() => done()) @@ -507,7 +558,8 @@ function makeVeeamRequest(params, callback) { }); }); afterEach(done => { - bucketUtil.empty(TEST_BUCKET) + bucketUtil + .empty(TEST_BUCKET) .then(() => s3.send(new DeleteBucketCommand({ Bucket: TEST_BUCKET }))) .then(() => done()) .catch(done); @@ -518,42 +570,101 @@ function makeVeeamRequest(params, callback) { ['.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/capacity.xml', testCapacity, testCapacityMd5], ].forEach(key => { it(`DELETE ${key[0]} should delete the XML file`, done => { - async.waterfall([ - next => makeVeeamRequest({ - method: 'PUT', - bucket: TEST_BUCKET, - objectKey: key[0], - headers: { - 'content-length': key[1].length, - 'content-md5': key[2], - 'x-scal-canonical-id': testArn, - }, - authCredentials: veeamAuthCredentials, - requestBody: key[1], - }, (err, response) => { - if (err) { - return done(err); - } - assert.strictEqual(response.statusCode, 200); - return next(); - }), - next => makeVeeamRequest({ - method: 'GET', - bucket: TEST_BUCKET, - objectKey: key[0], - headers: { - 'x-scal-canonical-id': testArn, - }, - authCredentials: veeamAuthCredentials, - }, (err, response) => { - if (err) { - return done(err); - } - assert.strictEqual(response.statusCode, 200); - assert.strictEqual(response.body.replaceAll(' ', ''), key[1].replaceAll(' ', '')); - return next(); - }), - next => makeVeeamRequest({ + async.waterfall( + [ + next => + makeVeeamRequest( + { + method: 'PUT', + bucket: TEST_BUCKET, + objectKey: key[0], + headers: { + 'content-length': key[1].length, + 'content-md5': key[2], + 'x-scal-canonical-id': testArn, + }, + authCredentials: veeamAuthCredentials, + requestBody: key[1], + }, + (err, response) => { + if (err) { + return done(err); + } + assert.strictEqual(response.statusCode, 200); + return next(); + }, + ), + next => + makeVeeamRequest( + { + method: 'GET', + bucket: TEST_BUCKET, + objectKey: key[0], + headers: { + 'x-scal-canonical-id': testArn, + }, + authCredentials: veeamAuthCredentials, + }, + (err, response) => { + if (err) { + return done(err); + } + assert.strictEqual(response.statusCode, 200); + assert.strictEqual(response.body.replaceAll(' ', ''), key[1].replaceAll(' ', '')); + return next(); + }, + ), + next => + makeVeeamRequest( + { + method: 'DELETE', + bucket: TEST_BUCKET, + objectKey: key[0], + headers: { + 'x-scal-canonical-id': testArn, + }, + authCredentials: veeamAuthCredentials, + }, + (err, response) => { + if (err) { + return done(err); + } + assert.strictEqual(response.statusCode, 204); + return next(); + }, + ), + next => + makeVeeamRequest( + { + method: 'GET', + bucket: TEST_BUCKET, + objectKey: key[0], + headers: { + 'x-scal-canonical-id': testArn, + }, + authCredentials: veeamAuthCredentials, + }, + err => { + assert.strictEqual(err.code, 'NoSuchKey'); + return next(); + }, + ), + ], + err => { + assert.ifError(err); + return done(); + }, + ); + }); + }); + + [ + ['.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/system.xml', testSystem, testSystemMd5], + ['.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/capacity.xml', testCapacity, testCapacityMd5], + ].forEach(key => { + it(`DELETE ${key[0]} should fail if XML doesn't exist yet`, done => + makeVeeamRequest( + { method: 'DELETE', bucket: TEST_BUCKET, objectKey: key[0], @@ -561,55 +672,18 @@ function makeVeeamRequest(params, callback) { 'x-scal-canonical-id': testArn, }, authCredentials: veeamAuthCredentials, - }, (err, response) => { - if (err) { - return done(err); - } - assert.strictEqual(response.statusCode, 204); - return next(); - }), - next => makeVeeamRequest({ - method: 'GET', - bucket: TEST_BUCKET, - objectKey: key[0], - headers: { - 'x-scal-canonical-id': testArn, - }, - authCredentials: veeamAuthCredentials, - }, err => { + }, + err => { assert.strictEqual(err.code, 'NoSuchKey'); - return next(); - }), - ], err => { - assert.ifError(err); - return done(); - }); - }); - }); - - [ - ['.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/system.xml', testSystem, testSystemMd5], - ['.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/capacity.xml', testCapacity, testCapacityMd5], - ].forEach(key => { - it(`DELETE ${key[0]} should fail if XML doesn't exist yet`, done => makeVeeamRequest({ - method: 'DELETE', - bucket: TEST_BUCKET, - objectKey: key[0], - headers: { - 'x-scal-canonical-id': testArn, - }, - authCredentials: veeamAuthCredentials, - }, err => { - assert.strictEqual(err.code, 'NoSuchKey'); - return done(); - })); + return done(); + }, + )); }); }); describe('veeam HEAD routes:', () => { beforeEach(done => { - bucketUtil = new BucketUtility( - 'default', { signatureVersion: 'v4' }); + bucketUtil = new BucketUtility('default', { signatureVersion: 'v4' }); s3 = bucketUtil.s3; s3.send(new CreateBucketCommand({ Bucket: TEST_BUCKET })) .then(() => done()) @@ -619,7 +693,8 @@ function makeVeeamRequest(params, callback) { }); }); afterEach(done => { - bucketUtil.empty(TEST_BUCKET) + bucketUtil + .empty(TEST_BUCKET) .then(() => s3.send(new DeleteBucketCommand({ Bucket: TEST_BUCKET }))) .then(() => done()) .catch(done); @@ -630,44 +705,55 @@ function makeVeeamRequest(params, callback) { ['.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/capacity.xml', testCapacity, testCapacityMd5], ].forEach(key => { it(`HEAD ${key[0]} should return the existing XML file metadata`, done => { - async.waterfall([ - next => makeVeeamRequest({ - method: 'PUT', - bucket: TEST_BUCKET, - objectKey: key[0], - headers: { - 'content-length': key[1].length, - 'content-md5': key[2], - 'x-scal-canonical-id': testArn, - }, - authCredentials: veeamAuthCredentials, - requestBody: key[1], - }, (err, response) => { - if (err) { - return done(err); - } - assert.strictEqual(response.statusCode, 200); - return next(); - }), - next => makeVeeamRequest({ - method: 'HEAD', - bucket: TEST_BUCKET, - objectKey: key[0], - headers: { - 'x-scal-canonical-id': testArn, - }, - authCredentials: veeamAuthCredentials, - }, (err, response) => { - if (err) { - return done(err); - } - assert.strictEqual(response.statusCode, 200); - return next(); - }), - ], err => { - assert.ifError(err); - return done(); - }); + async.waterfall( + [ + next => + makeVeeamRequest( + { + method: 'PUT', + bucket: TEST_BUCKET, + objectKey: key[0], + headers: { + 'content-length': key[1].length, + 'content-md5': key[2], + 'x-scal-canonical-id': testArn, + }, + authCredentials: veeamAuthCredentials, + requestBody: key[1], + }, + (err, response) => { + if (err) { + return done(err); + } + assert.strictEqual(response.statusCode, 200); + return next(); + }, + ), + next => + makeVeeamRequest( + { + method: 'HEAD', + bucket: TEST_BUCKET, + objectKey: key[0], + headers: { + 'x-scal-canonical-id': testArn, + }, + authCredentials: veeamAuthCredentials, + }, + (err, response) => { + if (err) { + return done(err); + } + assert.strictEqual(response.statusCode, 200); + return next(); + }, + ), + ], + err => { + assert.ifError(err); + return done(); + }, + ); }); }); @@ -675,28 +761,30 @@ function makeVeeamRequest(params, callback) { ['.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/system.xml', testSystem, testSystemMd5], ['.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/capacity.xml', testCapacity, testCapacityMd5], ].forEach(key => { - it(`HEAD ${key[0]} should fail if no data in bucket metadata`, done => makeVeeamRequest({ - method: 'HEAD', - bucket: TEST_BUCKET, - objectKey: key[0], - headers: { - 'x-scal-canonical-id': testArn, - }, - authCredentials: veeamAuthCredentials, - }, (err, res) => { - assert.strictEqual(res.statusCode, 404); - return done(); - })); + it(`HEAD ${key[0]} should fail if no data in bucket metadata`, done => + makeVeeamRequest( + { + method: 'HEAD', + bucket: TEST_BUCKET, + objectKey: key[0], + headers: { + 'x-scal-canonical-id': testArn, + }, + authCredentials: veeamAuthCredentials, + }, + (err, res) => { + assert.strictEqual(res.statusCode, 404); + return done(); + }, + )); }); }); }); - // TODO {test_debt} handle query params tests with signature (happy path) describe.skip('veeam LIST routes:', () => { beforeEach(done => { - bucketUtil = new BucketUtility( - 'default', { signatureVersion: 'v4' }); + bucketUtil = new BucketUtility('default', { signatureVersion: 'v4' }); s3 = bucketUtil.s3; s3.send(new CreateBucketCommand({ Bucket: TEST_BUCKET })) .then(() => done()) @@ -706,7 +794,8 @@ describe.skip('veeam LIST routes:', () => { }); }); afterEach(done => { - bucketUtil.empty(TEST_BUCKET) + bucketUtil + .empty(TEST_BUCKET) .then(() => s3.send(new DeleteBucketCommand({ Bucket: TEST_BUCKET }))) .then(() => done()) .catch(done); diff --git a/tests/unit/Config.js b/tests/unit/Config.js index 09d642f4fa..31a8586404 100644 --- a/tests/unit/Config.js +++ b/tests/unit/Config.js @@ -11,9 +11,7 @@ const { ConfigObject, } = require('../../lib/Config'); -const { - LOCATION_NAME_DMF, -} = require('../constants'); +const { LOCATION_NAME_DMF } = require('../constants'); const constants = require('../../constants'); const { ValidLifecycleRules: supportedLifecycleRules } = require('arsenal').models; @@ -25,15 +23,23 @@ describe('Config', () => { const setEnv = (key, value) => { if (key in process.env) { const v = process.env[key]; - envToRestore.push(() => { process.env[key] = v; }); + envToRestore.push(() => { + process.env[key] = v; + }); } else { - envToRestore.push(() => { delete process.env[key]; }); + envToRestore.push(() => { + delete process.env[key]; + }); } process.env[key] = value; }; - beforeEach(() => { envToRestore.length = 0; }); - afterEach(() => { envToRestore.reverse().forEach(cb => cb()); }); + beforeEach(() => { + envToRestore.length = 0; + }); + afterEach(() => { + envToRestore.reverse().forEach(cb => cb()); + }); it('should load default config.json without errors', done => { require('../../lib/Config'); @@ -56,7 +62,7 @@ describe('Config', () => { describe('azureGetStorageAccountName', () => { it('should return the azureStorageAccountName', done => { const accountName = azureGetStorageAccountName('us-west-1', { - azureStorageAccountName: 'someaccount' + azureStorageAccountName: 'someaccount', }); assert.deepStrictEqual(accountName, 'someaccount'); return done(); @@ -66,7 +72,7 @@ describe('Config', () => { setEnv('us-west-1_AZURE_STORAGE_ACCOUNT_NAME', 'other'); setEnv('fr-east-2_AZURE_STORAGE_ACCOUNT_NAME', 'wrong'); const accountName = azureGetStorageAccountName('us-west-1', { - azureStorageAccountName: 'someaccount' + azureStorageAccountName: 'someaccount', }); assert.deepStrictEqual(accountName, 'other'); return done(); @@ -109,7 +115,7 @@ describe('Config', () => { it('should return shared-key credentials with authMethod from details', () => { const creds = azureGetLocationCredentials('us-west-1', { authMode: 'shared-key', - ...locationDetails + ...locationDetails, }); assert.deepStrictEqual(creds, { authMethod: 'shared-key', @@ -150,7 +156,7 @@ describe('Config', () => { it('should return shared-access-signature-token credentials with authMethod from details', () => { const creds = azureGetLocationCredentials('us-west-1', { authMethod: 'shared-access-signature', - ...locationDetails + ...locationDetails, }); assert.deepStrictEqual(creds, { authMethod: 'shared-access-signature', @@ -197,7 +203,7 @@ describe('Config', () => { it('should return client-secret credentials with authMethod from details', () => { const creds = azureGetLocationCredentials('us-west-1', { authMethod: 'client-secret', - ...locationDetails + ...locationDetails, }); assert.deepStrictEqual(creds, { authMethod: 'client-secret', @@ -223,69 +229,54 @@ describe('Config', () => { it('should return account name from config', () => { setEnv('azurebackend_AZURE_STORAGE_ACCOUNT_NAME', ''); const config = new ConfigObject(); - assert.deepStrictEqual( - config.getAzureStorageAccountName('azurebackend'), - 'fakeaccountname' - ); + assert.deepStrictEqual(config.getAzureStorageAccountName('azurebackend'), 'fakeaccountname'); }); it('should return account name from env', () => { setEnv('azurebackend_AZURE_STORAGE_ACCOUNT_NAME', 'foooo'); const config = new ConfigObject(); - assert.deepStrictEqual( - config.getAzureStorageAccountName('azurebackend'), - 'foooo' - ); + assert.deepStrictEqual(config.getAzureStorageAccountName('azurebackend'), 'foooo'); }); it('should return account name from shared-access-signature auth', () => { setEnv('S3_LOCATION_FILE', 'tests/locationConfig/locationConfigTests.json'); const config = new ConfigObject(); - assert.deepStrictEqual( - config.getAzureStorageAccountName('azurebackend3'), - 'fakeaccountname3' - ); + assert.deepStrictEqual(config.getAzureStorageAccountName('azurebackend3'), 'fakeaccountname3'); }); it('should return account name from client-secret auth', () => { setEnv('S3_LOCATION_FILE', 'tests/locationConfig/locationConfigTests.json'); const config = new ConfigObject(); - assert.deepStrictEqual( - config.getAzureStorageAccountName('azurebackend4'), - 'fakeaccountname4', - ); + assert.deepStrictEqual(config.getAzureStorageAccountName('azurebackend4'), 'fakeaccountname4'); }); it('should return account name from endpoint', () => { setEnv('S3_LOCATION_FILE', 'tests/locationConfig/locationConfigTests.json'); const config = new ConfigObject(); - assert.deepStrictEqual( - config.getAzureStorageAccountName('azuritebackend'), - 'myfakeaccount', - ); + assert.deepStrictEqual(config.getAzureStorageAccountName('azuritebackend'), 'myfakeaccount'); }); }); describe('locationConstraintAssert', () => { const memLocation = { - 'details': {}, - 'isCold': false, - 'isTransient': false, - 'legacyAwsBehavior': false, - 'locationType': 'location-mem-v1', - 'objectId': 'a9d9b632-5fa5-11ef-8715-b21941dbc3ea', - 'type': 'mem', + details: {}, + isCold: false, + isTransient: false, + legacyAwsBehavior: false, + locationType: 'location-mem-v1', + objectId: 'a9d9b632-5fa5-11ef-8715-b21941dbc3ea', + type: 'mem', }; it('should parse tlp location', () => { const locationConstraints = { 'dmf-1': { - 'details': {}, - 'isCold': true, - 'legacyAwsBehavior': false, - 'locationType': LOCATION_NAME_DMF, - 'objectId': 'b9d9b632-5fa5-11ef-8715-b21941dbc3ea', - 'type': 'tlp' + details: {}, + isCold: true, + legacyAwsBehavior: false, + locationType: LOCATION_NAME_DMF, + objectId: 'b9d9b632-5fa5-11ef-8715-b21941dbc3ea', + type: 'tlp', }, 'us-east-1': memLocation, }; @@ -295,12 +286,12 @@ describe('Config', () => { it('should fail tlp location is not cold', () => { const locationConstraints = { 'dmf-1': { - 'details': {}, - 'isCold': false, - 'legacyAwsBehavior': false, - 'locationType': LOCATION_NAME_DMF, - 'objectId': 'b9d9b632-5fa5-11ef-8715-b21941dbc3ea', - 'type': 'tlp' + details: {}, + isCold: false, + legacyAwsBehavior: false, + locationType: LOCATION_NAME_DMF, + objectId: 'b9d9b632-5fa5-11ef-8715-b21941dbc3ea', + type: 'tlp', }, 'us-east-1': memLocation, }; @@ -310,14 +301,14 @@ describe('Config', () => { it('should fail if tlp location has details', () => { const locationConstraints = { 'dmf-1': { - 'details': { - 'endpoint': 'http://localhost:8000', + details: { + endpoint: 'http://localhost:8000', }, - 'isCold': true, - 'legacyAwsBehavior': false, - 'locationType': LOCATION_NAME_DMF, - 'objectId': 'b9d9b632-5fa5-11ef-8715-b21941dbc3ea', - 'type': 'tlp' + isCold: true, + legacyAwsBehavior: false, + locationType: LOCATION_NAME_DMF, + objectId: 'b9d9b632-5fa5-11ef-8715-b21941dbc3ea', + type: 'tlp', }, 'us-east-1': memLocation, }; @@ -513,8 +504,7 @@ describe('Config', () => { before(() => { oldConfig = process.env.S3_CONFIG_FILE; - process.env.S3_CONFIG_FILE = - 'tests/unit/testConfigs/allOptsConfig/config.json'; + process.env.S3_CONFIG_FILE = 'tests/unit/testConfigs/allOptsConfig/config.json'; }); after(() => { @@ -524,13 +514,10 @@ describe('Config', () => { it('should set up scuba', () => { const config = new ConfigObject(); - assert.deepStrictEqual( - config.scuba, - { - host: 'localhost', - port: 8100, - }, - ); + assert.deepStrictEqual(config.scuba, { + host: 'localhost', + port: 8100, + }); }); it('should use environment variables for scuba', () => { @@ -539,13 +526,10 @@ describe('Config', () => { const config = new ConfigObject(); - assert.deepStrictEqual( - config.scuba, - { - host: 'scubahost', - port: 1234, - }, - ); + assert.deepStrictEqual(config.scuba, { + host: 'scubahost', + port: 1234, + }); }); }); @@ -554,8 +538,7 @@ describe('Config', () => { before(() => { oldConfig = process.env.S3_CONFIG_FILE; - process.env.S3_CONFIG_FILE = - 'tests/unit/testConfigs/allOptsConfig/config.json'; + process.env.S3_CONFIG_FILE = 'tests/unit/testConfigs/allOptsConfig/config.json'; }); after(() => { @@ -565,13 +548,10 @@ describe('Config', () => { it('should set up quota', () => { const config = new ConfigObject(); - assert.deepStrictEqual( - config.quota, - { - maxStaleness: 24 * 60 * 60 * 1000, - enableInflights: false, - }, - ); + assert.deepStrictEqual(config.quota, { + maxStaleness: 24 * 60 * 60 * 1000, + enableInflights: false, + }); }); it('should use environment variables for scuba', () => { @@ -580,13 +560,10 @@ describe('Config', () => { const config = new ConfigObject(); - assert.deepStrictEqual( - config.quota, - { - maxStaleness: 1234, - enableInflights: true, - }, - ); + assert.deepStrictEqual(config.quota, { + maxStaleness: 1234, + enableInflights: true, + }); }); it('should use the default if the maxStaleness is not a number', () => { @@ -595,13 +572,10 @@ describe('Config', () => { const config = new ConfigObject(); - assert.deepStrictEqual( - config.quota, - { - maxStaleness: 24 * 60 * 60 * 1000, - enableInflights: true, - }, - ); + assert.deepStrictEqual(config.quota, { + maxStaleness: 24 * 60 * 60 * 1000, + enableInflights: true, + }); }); }); @@ -610,8 +584,7 @@ describe('Config', () => { before(() => { oldConfig = process.env.S3_CONFIG_FILE; - process.env.S3_CONFIG_FILE = - 'tests/unit/testConfigs/allOptsConfig/config.json'; + process.env.S3_CONFIG_FILE = 'tests/unit/testConfigs/allOptsConfig/config.json'; }); after(() => { @@ -621,35 +594,29 @@ describe('Config', () => { it('should set up utapi local cache', () => { const config = new ConfigObject(); - assert.deepStrictEqual( - config.localCache, - { name: 'zenko', sentinels: [{ host: 'localhost', port: 6379 }] }, - ); - assert.deepStrictEqual( - config.utapi.localCache, - config.localCache, - ); + assert.deepStrictEqual(config.localCache, { + name: 'zenko', + sentinels: [{ host: 'localhost', port: 6379 }], + }); + assert.deepStrictEqual(config.utapi.localCache, config.localCache); }); it('should set up utapi redis', () => { const config = new ConfigObject(); - assert.deepStrictEqual( - config.utapi.redis, - { - host: 'localhost', - port: 6379, - retry: { - connectBackoff: { - min: 10, - max: 1000, - factor: 1.5, - jitter: 0.1, - deadline: 10000, - }, + assert.deepStrictEqual(config.utapi.redis, { + host: 'localhost', + port: 6379, + retry: { + connectBackoff: { + min: 10, + max: 1000, + factor: 1.5, + jitter: 0.1, + deadline: 10000, }, }, - ); + }); }); }); @@ -674,11 +641,7 @@ describe('Config', () => { }); it('should return the rules provided when they are valid', () => { - const rules = [ - 'Expiration', - 'NoncurrentVersionExpiration', - 'AbortIncompleteMultipartUpload', - ]; + const rules = ['Expiration', 'NoncurrentVersionExpiration', 'AbortIncompleteMultipartUpload']; const parsedRules = parseSupportedLifecycleRules(rules); assert.deepStrictEqual(parsedRules, rules); }); @@ -837,8 +800,7 @@ describe('Config', () => { .withArgs(sinon.match(/\/config\.json$/)) .returns(JSON.stringify({ ...defaultConfig, instanceId: 'test' })); // For all other files, use the original readFileSync - readFileSyncStub - .callsFake((filePath, ...args) => originalReadFileSync(filePath, ...args)); + readFileSyncStub.callsFake((filePath, ...args) => originalReadFileSync(filePath, ...args)); // Create a new ConfigObject instance const config = new ConfigObject(); assert.strictEqual(config.instanceId, 'test'); @@ -853,8 +815,7 @@ describe('Config', () => { .withArgs(sinon.match(/\/config\.json$/)) .returns(JSON.stringify({ ...defaultConfig, instanceId: 1234 })); // For all other files, use the original readFileSync - readFileSyncStub - .callsFake((filePath, ...args) => originalReadFileSync(filePath, ...args)); + readFileSyncStub.callsFake((filePath, ...args) => originalReadFileSync(filePath, ...args)); // Create a new ConfigObject instance assert.throws(() => new ConfigObject()); }); @@ -894,14 +855,14 @@ describe('Config', () => { const multiObjectDeleteSize = 42; const modifiedConfig = { ...defaultConfig, - apiBodySizeLimits: { 'multiObjectDelete': multiObjectDeleteSize }, + apiBodySizeLimits: { multiObjectDelete: multiObjectDeleteSize }, }; readFileStub.withArgs(sinon.match(/config.json$/)).returns(JSON.stringify(modifiedConfig)); const config = new ConfigObject(); assert.deepStrictEqual(config.apiBodySizeLimits, { - 'multiObjectDelete': multiObjectDeleteSize, // Configured: overwrites default - 'bucketPutPolicy': constants.defaultApiBodySizeLimits['bucketPutPolicy'], // Not configured: default + multiObjectDelete: multiObjectDeleteSize, // Configured: overwrites default + bucketPutPolicy: constants.defaultApiBodySizeLimits['bucketPutPolicy'], // Not configured: default }); }); @@ -914,7 +875,7 @@ describe('Config', () => { assert.throws( () => new ConfigObject(), - /bad config: apiBodySizeLimits for "anApiNotSetInConstants.js" cannot be configured/ + /bad config: apiBodySizeLimits for "anApiNotSetInConstants.js" cannot be configured/, ); }); }); @@ -923,21 +884,21 @@ describe('Config', () => { it('should replace default values with new values', () => { const newConfig = { integrityChecks: { - 'bucketPutACL': false, - 'bucketPutCors': false, - 'bucketPutEncryption': false, - 'bucketPutLifecycle': false, - 'bucketPutNotification': false, - 'bucketPutObjectLock': false, - 'bucketPutPolicy': false, - 'bucketPutReplication': false, - 'bucketPutVersioning': false, - 'bucketPutWebsite': false, - 'multiObjectDelete': false, - 'objectPutACL': false, - 'objectPutLegalHold': false, - 'objectPutTagging': false, - 'objectPutRetention': false, + bucketPutACL: false, + bucketPutCors: false, + bucketPutEncryption: false, + bucketPutLifecycle: false, + bucketPutNotification: false, + bucketPutObjectLock: false, + bucketPutPolicy: false, + bucketPutReplication: false, + bucketPutVersioning: false, + bucketPutWebsite: false, + multiObjectDelete: false, + objectPutACL: false, + objectPutLegalHold: false, + objectPutTagging: false, + objectPutRetention: false, }, }; diff --git a/tests/unit/api/apiUtils/authorization/aclChecks.js b/tests/unit/api/apiUtils/authorization/aclChecks.js index 6bf34850b6..0821d5acc4 100644 --- a/tests/unit/api/apiUtils/authorization/aclChecks.js +++ b/tests/unit/api/apiUtils/authorization/aclChecks.js @@ -1,7 +1,9 @@ const assert = require('assert'); -const { isServiceAccount, getServiceAccountProperties } = - require('../../../../../lib/api/apiUtils/authorization/permissionChecks'); +const { + isServiceAccount, + getServiceAccountProperties, +} = require('../../../../../lib/api/apiUtils/authorization/permissionChecks'); describe('aclChecks', () => { it('should return whether a canonical ID is a service account', () => { @@ -12,15 +14,11 @@ describe('aclChecks', () => { }); it('should return properties of a service account by canonical ID', () => { - assert.strictEqual( - getServiceAccountProperties('abcdefghijkl'), undefined); - assert.strictEqual( - getServiceAccountProperties('abcdefghijkl/notaservice'), undefined); - assert.deepStrictEqual( - getServiceAccountProperties('abcdefghijkl/lifecycle'), {}); - assert.deepStrictEqual( - getServiceAccountProperties('abcdefghijkl/md-ingestion'), { - canReplicate: true, - }); + assert.strictEqual(getServiceAccountProperties('abcdefghijkl'), undefined); + assert.strictEqual(getServiceAccountProperties('abcdefghijkl/notaservice'), undefined); + assert.deepStrictEqual(getServiceAccountProperties('abcdefghijkl/lifecycle'), {}); + assert.deepStrictEqual(getServiceAccountProperties('abcdefghijkl/md-ingestion'), { + canReplicate: true, + }); }); }); diff --git a/tests/unit/api/apiUtils/authorization/prepareRequestContexts.js b/tests/unit/api/apiUtils/authorization/prepareRequestContexts.js index f2573a1b1a..bec694968d 100644 --- a/tests/unit/api/apiUtils/authorization/prepareRequestContexts.js +++ b/tests/unit/api/apiUtils/authorization/prepareRequestContexts.js @@ -1,15 +1,15 @@ const assert = require('assert'); const DummyRequest = require('../../../DummyRequest'); -const prepareRequestContexts = - require('../../../../../lib/api/apiUtils/authorization/prepareRequestContexts.js'); - -const makeRequest = (headers, query) => new DummyRequest({ - headers, - url: '/', - parsedHost: 'localhost', - socket: {}, - query, -}); +const prepareRequestContexts = require('../../../../../lib/api/apiUtils/authorization/prepareRequestContexts.js'); + +const makeRequest = (headers, query) => + new DummyRequest({ + headers, + url: '/', + parsedHost: 'localhost', + socket: {}, + query, + }); const sourceBucket = 'bucketsource'; const sourceObject = 'objectsource'; const sourceVersionId = 'vid1'; @@ -18,76 +18,79 @@ describe('prepareRequestContexts', () => { it('should return s3:DeleteObject if multiObjectDelete method', () => { const apiMethod = 'multiObjectDelete'; const request = makeRequest(); - const results = prepareRequestContexts(apiMethod, request, sourceBucket, - sourceObject, sourceVersionId); + const results = prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId); assert.strictEqual(results.length, 1); const expectedAction = 's3:DeleteObject'; assert.strictEqual(results[0].getAction(), expectedAction); }); - it('should return s3:PutObjectVersion request context action for objectPut method with x-scal-s3-version-id' + - ' header', () => { - const apiMethod = 'objectPut'; - const request = makeRequest({ - 'x-scal-s3-version-id': 'vid', - }); - const results = prepareRequestContexts(apiMethod, request, sourceBucket, - sourceObject, sourceVersionId); - - assert.strictEqual(results.length, 1); - const expectedAction = 's3:PutObjectVersion'; - assert.strictEqual(results[0].getAction(), expectedAction); - }); + it( + 'should return s3:PutObjectVersion request context action for objectPut method with x-scal-s3-version-id' + + ' header', + () => { + const apiMethod = 'objectPut'; + const request = makeRequest({ + 'x-scal-s3-version-id': 'vid', + }); + const results = prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId); - it('should return s3:PutObjectVersion request context action for objectPut method with empty x-scal-s3-version-id' + - ' header', () => { - const apiMethod = 'objectPut'; - const request = makeRequest({ - 'x-scal-s3-version-id': '', - }); - const results = prepareRequestContexts(apiMethod, request, sourceBucket, - sourceObject, sourceVersionId); + assert.strictEqual(results.length, 1); + const expectedAction = 's3:PutObjectVersion'; + assert.strictEqual(results[0].getAction(), expectedAction); + }, + ); + + it( + 'should return s3:PutObjectVersion request context action for objectPut method with empty x-scal-s3-version-id' + + ' header', + () => { + const apiMethod = 'objectPut'; + const request = makeRequest({ + 'x-scal-s3-version-id': '', + }); + const results = prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId); - assert.strictEqual(results.length, 1); - const expectedAction = 's3:PutObjectVersion'; - assert.strictEqual(results[0].getAction(), expectedAction); - }); + assert.strictEqual(results.length, 1); + const expectedAction = 's3:PutObjectVersion'; + assert.strictEqual(results[0].getAction(), expectedAction); + }, + ); it('should return s3:PutObject request context action for objectPut method and no header', () => { const apiMethod = 'objectPut'; const request = makeRequest({}); - const results = prepareRequestContexts(apiMethod, request, sourceBucket, - sourceObject, sourceVersionId); + const results = prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId); assert.strictEqual(results.length, 1); const expectedAction = 's3:PutObject'; assert.strictEqual(results[0].getAction(), expectedAction); }); - it('should return s3:PutObject and s3:PutObjectTagging actions for objectPut method with' + - ' x-amz-tagging header', () => { - const apiMethod = 'objectPut'; - const request = makeRequest({ - 'x-amz-tagging': 'key1=value1&key2=value2', - }); - const results = prepareRequestContexts(apiMethod, request, sourceBucket, - sourceObject, sourceVersionId); + it( + 'should return s3:PutObject and s3:PutObjectTagging actions for objectPut method with' + + ' x-amz-tagging header', + () => { + const apiMethod = 'objectPut'; + const request = makeRequest({ + 'x-amz-tagging': 'key1=value1&key2=value2', + }); + const results = prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId); - assert.strictEqual(results.length, 2); - const expectedAction1 = 's3:PutObject'; - const expectedAction2 = 's3:PutObjectTagging'; - assert.strictEqual(results[0].getAction(), expectedAction1); - assert.strictEqual(results[1].getAction(), expectedAction2); - }); + assert.strictEqual(results.length, 2); + const expectedAction1 = 's3:PutObject'; + const expectedAction2 = 's3:PutObjectTagging'; + assert.strictEqual(results[0].getAction(), expectedAction1); + assert.strictEqual(results[1].getAction(), expectedAction2); + }, + ); it('should return s3:PutObject and s3:PutObjectAcl actions for objectPut method with ACL header', () => { const apiMethod = 'objectPut'; const request = makeRequest({ 'x-amz-acl': 'private', }); - const results = prepareRequestContexts(apiMethod, request, sourceBucket, - sourceObject, sourceVersionId); + const results = prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId); assert.strictEqual(results.length, 2); const expectedAction1 = 's3:PutObject'; @@ -98,10 +101,8 @@ describe('prepareRequestContexts', () => { it('should return s3:GetObject for headObject', () => { const apiMethod = 'objectHead'; - const request = makeRequest({ - }); - const results = prepareRequestContexts(apiMethod, request, sourceBucket, - sourceObject, sourceVersionId); + const request = makeRequest({}); + const results = prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId); assert.strictEqual(results.length, 1); assert.strictEqual(results[0].getAction(), 's3:GetObject'); @@ -112,43 +113,46 @@ describe('prepareRequestContexts', () => { const request = makeRequest({ 'x-amz-version-id': '0987654323456789', }); - const results = prepareRequestContexts(apiMethod, request, sourceBucket, - sourceObject, sourceVersionId); + const results = prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId); assert.strictEqual(results.length, 2); assert.strictEqual(results[0].getAction(), 's3:GetObject'); assert.strictEqual(results[1].getAction(), 's3:GetObjectVersion'); }); - it('should return s3:GetObject and scality:GetObjectArchiveInfo for headObject ' + - 'with x-amz-scal-archive-info header', () => { - const apiMethod = 'objectHead'; - const request = makeRequest({ - 'x-amz-scal-archive-info': 'true', - }); - const results = prepareRequestContexts(apiMethod, request, sourceBucket, - sourceObject, sourceVersionId); - - assert.strictEqual(results.length, 2); - assert.strictEqual(results[0].getAction(), 's3:GetObject'); - assert.strictEqual(results[1].getAction(), 'scality:GetObjectArchiveInfo'); - }); - - it('should return s3:GetObject, s3:GetObjectVersion and scality:GetObjectArchiveInfo ' + - ' for headObject with x-amz-scal-archive-info header', () => { - const apiMethod = 'objectHead'; - const request = makeRequest({ - 'x-amz-version-id': '0987654323456789', - 'x-amz-scal-archive-info': 'true', - }); - const results = prepareRequestContexts(apiMethod, request, sourceBucket, - sourceObject, sourceVersionId); + it( + 'should return s3:GetObject and scality:GetObjectArchiveInfo for headObject ' + + 'with x-amz-scal-archive-info header', + () => { + const apiMethod = 'objectHead'; + const request = makeRequest({ + 'x-amz-scal-archive-info': 'true', + }); + const results = prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId); + + assert.strictEqual(results.length, 2); + assert.strictEqual(results[0].getAction(), 's3:GetObject'); + assert.strictEqual(results[1].getAction(), 'scality:GetObjectArchiveInfo'); + }, + ); + + it( + 'should return s3:GetObject, s3:GetObjectVersion and scality:GetObjectArchiveInfo ' + + ' for headObject with x-amz-scal-archive-info header', + () => { + const apiMethod = 'objectHead'; + const request = makeRequest({ + 'x-amz-version-id': '0987654323456789', + 'x-amz-scal-archive-info': 'true', + }); + const results = prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId); - assert.strictEqual(results.length, 3); - assert.strictEqual(results[0].getAction(), 's3:GetObject'); - assert.strictEqual(results[1].getAction(), 's3:GetObjectVersion'); - assert.strictEqual(results[2].getAction(), 'scality:GetObjectArchiveInfo'); - }); + assert.strictEqual(results.length, 3); + assert.strictEqual(results[0].getAction(), 's3:GetObject'); + assert.strictEqual(results[1].getAction(), 's3:GetObjectVersion'); + assert.strictEqual(results[2].getAction(), 'scality:GetObjectArchiveInfo'); + }, + ); it('should return s3:PutObjectRetention with header x-amz-object-lock-mode', () => { const apiMethod = 'objectPut'; @@ -165,45 +169,54 @@ describe('prepareRequestContexts', () => { assert.strictEqual(results[1].getAction(), expectedAction2); }); - it('should return s3:PutObjectRetention and s3:BypassGovernanceRetention for objectPut ' + - 'with header x-amz-bypass-governance-retention', () => { - const apiMethod = 'objectPut'; - const request = makeRequest({ - 'x-amz-object-lock-mode': 'GOVERNANCE', - 'x-amz-object-lock-retain-until-date': '2021-12-31T23:59:59.000Z', - 'x-amz-bypass-governance-retention': 'true', - }); - const results = prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId); - - assert.strictEqual(results.length, 3); - const expectedAction1 = 's3:PutObject'; - const expectedAction2 = 's3:PutObjectRetention'; - const expectedAction3 = 's3:BypassGovernanceRetention'; - assert.strictEqual(results[0].getAction(), expectedAction1); - assert.strictEqual(results[1].getAction(), expectedAction2); - assert.strictEqual(results[2].getAction(), expectedAction3); - }); - - it('should return s3:PutObjectRetention and s3:BypassGovernanceRetention for objectPut ' + - 'with header x-amz-bypass-governance-retention with version id specified', () => { - const apiMethod = 'objectPut'; - const request = makeRequest({ - 'x-amz-object-lock-mode': 'GOVERNANCE', - 'x-amz-object-lock-retain-until-date': '2021-12-31T23:59:59.000Z', - 'x-amz-bypass-governance-retention': 'true', - }, { - versionId: 'vid1', - }); - const results = prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId); - - assert.strictEqual(results.length, 3); - const expectedAction1 = 's3:PutObject'; - const expectedAction2 = 's3:PutObjectRetention'; - const expectedAction3 = 's3:BypassGovernanceRetention'; - assert.strictEqual(results[0].getAction(), expectedAction1); - assert.strictEqual(results[1].getAction(), expectedAction2); - assert.strictEqual(results[2].getAction(), expectedAction3); - }); + it( + 'should return s3:PutObjectRetention and s3:BypassGovernanceRetention for objectPut ' + + 'with header x-amz-bypass-governance-retention', + () => { + const apiMethod = 'objectPut'; + const request = makeRequest({ + 'x-amz-object-lock-mode': 'GOVERNANCE', + 'x-amz-object-lock-retain-until-date': '2021-12-31T23:59:59.000Z', + 'x-amz-bypass-governance-retention': 'true', + }); + const results = prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId); + + assert.strictEqual(results.length, 3); + const expectedAction1 = 's3:PutObject'; + const expectedAction2 = 's3:PutObjectRetention'; + const expectedAction3 = 's3:BypassGovernanceRetention'; + assert.strictEqual(results[0].getAction(), expectedAction1); + assert.strictEqual(results[1].getAction(), expectedAction2); + assert.strictEqual(results[2].getAction(), expectedAction3); + }, + ); + + it( + 'should return s3:PutObjectRetention and s3:BypassGovernanceRetention for objectPut ' + + 'with header x-amz-bypass-governance-retention with version id specified', + () => { + const apiMethod = 'objectPut'; + const request = makeRequest( + { + 'x-amz-object-lock-mode': 'GOVERNANCE', + 'x-amz-object-lock-retain-until-date': '2021-12-31T23:59:59.000Z', + 'x-amz-bypass-governance-retention': 'true', + }, + { + versionId: 'vid1', + }, + ); + const results = prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId); + + assert.strictEqual(results.length, 3); + const expectedAction1 = 's3:PutObject'; + const expectedAction2 = 's3:PutObjectRetention'; + const expectedAction3 = 's3:BypassGovernanceRetention'; + assert.strictEqual(results[0].getAction(), expectedAction1); + assert.strictEqual(results[1].getAction(), expectedAction2); + assert.strictEqual(results[2].getAction(), expectedAction3); + }, + ); it('should return s3:PutObjectRetention with header x-amz-object-lock-mode for objectPutRetention action', () => { const apiMethod = 'objectPutRetention'; @@ -218,47 +231,55 @@ describe('prepareRequestContexts', () => { assert.strictEqual(results[0].getAction(), expectedAction); }); - it('should return s3:PutObjectRetention and s3:BypassGovernanceRetention for objectPutRetention ' + - 'with header x-amz-bypass-governance-retention', () => { - const apiMethod = 'objectPutRetention'; - const request = makeRequest({ - 'x-amz-object-lock-mode': 'GOVERNANCE', - 'x-amz-object-lock-retain-until-date': '2021-12-31T23:59:59.000Z', - 'x-amz-bypass-governance-retention': 'true', - }); - const results = prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId); - - assert.strictEqual(results.length, 2); - const expectedAction1 = 's3:PutObjectRetention'; - const expectedAction2 = 's3:BypassGovernanceRetention'; - assert.strictEqual(results[0].getAction(), expectedAction1); - assert.strictEqual(results[1].getAction(), expectedAction2); - }); - - it('should return s3:PutObjectRetention and s3:BypassGovernanceRetention for objectPutRetention ' + - 'with header x-amz-bypass-governance-retention with version id specified', () => { - const apiMethod = 'objectPutRetention'; - const request = makeRequest({ - 'x-amz-object-lock-mode': 'GOVERNANCE', - 'x-amz-object-lock-retain-until-date': '2021-12-31T23:59:59.000Z', - 'x-amz-bypass-governance-retention': 'true', - }, { - versionId: 'vid1', - }); - const results = prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId); - - assert.strictEqual(results.length, 2); - const expectedAction1 = 's3:PutObjectRetention'; - const expectedAction2 = 's3:BypassGovernanceRetention'; - assert.strictEqual(results[0].getAction(), expectedAction1); - assert.strictEqual(results[1].getAction(), expectedAction2); - }); + it( + 'should return s3:PutObjectRetention and s3:BypassGovernanceRetention for objectPutRetention ' + + 'with header x-amz-bypass-governance-retention', + () => { + const apiMethod = 'objectPutRetention'; + const request = makeRequest({ + 'x-amz-object-lock-mode': 'GOVERNANCE', + 'x-amz-object-lock-retain-until-date': '2021-12-31T23:59:59.000Z', + 'x-amz-bypass-governance-retention': 'true', + }); + const results = prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId); + + assert.strictEqual(results.length, 2); + const expectedAction1 = 's3:PutObjectRetention'; + const expectedAction2 = 's3:BypassGovernanceRetention'; + assert.strictEqual(results[0].getAction(), expectedAction1); + assert.strictEqual(results[1].getAction(), expectedAction2); + }, + ); + + it( + 'should return s3:PutObjectRetention and s3:BypassGovernanceRetention for objectPutRetention ' + + 'with header x-amz-bypass-governance-retention with version id specified', + () => { + const apiMethod = 'objectPutRetention'; + const request = makeRequest( + { + 'x-amz-object-lock-mode': 'GOVERNANCE', + 'x-amz-object-lock-retain-until-date': '2021-12-31T23:59:59.000Z', + 'x-amz-bypass-governance-retention': 'true', + }, + { + versionId: 'vid1', + }, + ); + const results = prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId); + + assert.strictEqual(results.length, 2); + const expectedAction1 = 's3:PutObjectRetention'; + const expectedAction2 = 's3:BypassGovernanceRetention'; + assert.strictEqual(results[0].getAction(), expectedAction1); + assert.strictEqual(results[1].getAction(), expectedAction2); + }, + ); it('should return s3:DeleteObject for objectDelete method', () => { const apiMethod = 'objectDelete'; const request = makeRequest(); - const results = prepareRequestContexts(apiMethod, request, sourceBucket, - sourceObject, sourceVersionId); + const results = prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId); assert.strictEqual(results.length, 1); assert.strictEqual(results[0].getAction(), 's3:DeleteObject'); @@ -266,82 +287,94 @@ describe('prepareRequestContexts', () => { it('should return s3:DeleteObjectVersion for objectDelete method with version id specified', () => { const apiMethod = 'objectDelete'; - const request = makeRequest({}, { - versionId: 'vid1', - }); - const results = prepareRequestContexts(apiMethod, request, sourceBucket, - sourceObject, sourceVersionId); + const request = makeRequest( + {}, + { + versionId: 'vid1', + }, + ); + const results = prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId); assert.strictEqual(results.length, 1); assert.strictEqual(results[0].getAction(), 's3:DeleteObjectVersion'); }); // Now it shuld include the bypass header if set - it('should return s3:DeleteObjectVersion and s3:BypassGovernanceRetention for objectDelete method ' + - 'with version id specified and x-amz-bypass-governance-retention header', () => { - const apiMethod = 'objectDelete'; - const request = makeRequest({ - 'x-amz-bypass-governance-retention': 'true', - }, { - versionId: 'vid1', - }); - const results = prepareRequestContexts(apiMethod, request, sourceBucket, - sourceObject, sourceVersionId); - - assert.strictEqual(results.length, 2); - const expectedAction1 = 's3:DeleteObjectVersion'; - const expectedAction2 = 's3:BypassGovernanceRetention'; - assert.strictEqual(results[0].getAction(), expectedAction1); - assert.strictEqual(results[1].getAction(), expectedAction2); - }); + it( + 'should return s3:DeleteObjectVersion and s3:BypassGovernanceRetention for objectDelete method ' + + 'with version id specified and x-amz-bypass-governance-retention header', + () => { + const apiMethod = 'objectDelete'; + const request = makeRequest( + { + 'x-amz-bypass-governance-retention': 'true', + }, + { + versionId: 'vid1', + }, + ); + const results = prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId); + + assert.strictEqual(results.length, 2); + const expectedAction1 = 's3:DeleteObjectVersion'; + const expectedAction2 = 's3:BypassGovernanceRetention'; + assert.strictEqual(results[0].getAction(), expectedAction1); + assert.strictEqual(results[1].getAction(), expectedAction2); + }, + ); // When there is no version ID, AWS does not return any error if the object // is locked, but creates a delete marker - it('should only return s3:DeleteObject for objectDelete method ' + - 'with x-amz-bypass-governance-retention header and no version id', () => { - const apiMethod = 'objectDelete'; - const request = makeRequest({ - 'x-amz-bypass-governance-retention': 'true', - }); - const results = prepareRequestContexts(apiMethod, request, sourceBucket, - sourceObject, sourceVersionId); - - assert.strictEqual(results.length, 1); - const expectedAction = 's3:DeleteObject'; - assert.strictEqual(results[0].getAction(), expectedAction); - }); - - ['initiateMultipartUpload', 'objectPutPart', 'completeMultipartUpload'].forEach(apiMethod => { - it(`should return s3:PutObjectVersion request context action for ${apiMethod} method ` + - 'with x-scal-s3-version-id header', () => { + it( + 'should only return s3:DeleteObject for objectDelete method ' + + 'with x-amz-bypass-governance-retention header and no version id', + () => { + const apiMethod = 'objectDelete'; const request = makeRequest({ - 'x-scal-s3-version-id': '', + 'x-amz-bypass-governance-retention': 'true', }); - const results = prepareRequestContexts(apiMethod, request, sourceBucket, - sourceObject, sourceVersionId); + const results = prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId); assert.strictEqual(results.length, 1); - const expectedAction = 's3:PutObjectVersion'; + const expectedAction = 's3:DeleteObject'; assert.strictEqual(results[0].getAction(), expectedAction); - }); + }, + ); - it(`should return s3:PutObjectVersion request context action for ${apiMethod} method` + - 'with empty x-scal-s3-version-id header', () => { - const request = makeRequest({ - 'x-scal-s3-version-id': '', - }); - const results = prepareRequestContexts(apiMethod, request, sourceBucket, - sourceObject, sourceVersionId); + ['initiateMultipartUpload', 'objectPutPart', 'completeMultipartUpload'].forEach(apiMethod => { + it( + `should return s3:PutObjectVersion request context action for ${apiMethod} method ` + + 'with x-scal-s3-version-id header', + () => { + const request = makeRequest({ + 'x-scal-s3-version-id': '', + }); + const results = prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId); - assert.strictEqual(results.length, 1); - const expectedAction = 's3:PutObjectVersion'; - assert.strictEqual(results[0].getAction(), expectedAction); - }); + assert.strictEqual(results.length, 1); + const expectedAction = 's3:PutObjectVersion'; + assert.strictEqual(results[0].getAction(), expectedAction); + }, + ); + + it( + `should return s3:PutObjectVersion request context action for ${apiMethod} method` + + 'with empty x-scal-s3-version-id header', + () => { + const request = makeRequest({ + 'x-scal-s3-version-id': '', + }); + const results = prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId); + + assert.strictEqual(results.length, 1); + const expectedAction = 's3:PutObjectVersion'; + assert.strictEqual(results[0].getAction(), expectedAction); + }, + ); it(`should return s3:PutObject request context action for ${apiMethod} method and no header`, () => { const request = makeRequest({}); - const results = prepareRequestContexts(apiMethod, request, sourceBucket, - sourceObject, sourceVersionId); + const results = prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId); assert.strictEqual(results.length, 1); const expectedAction = 's3:PutObject'; @@ -463,7 +496,7 @@ describe('prepareRequestContexts', () => { const apiMethod = 'objectGetAttributes'; const request = makeRequest( { 'x-amz-object-attributes': 'x-amz-meta-department' }, - { versionId: '0987654323456789' } + { versionId: '0987654323456789' }, ); const results = prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId); diff --git a/tests/unit/api/apiUtils/coldStorage.js b/tests/unit/api/apiUtils/coldStorage.js index d38a6962b8..a3b9c78f58 100644 --- a/tests/unit/api/apiUtils/coldStorage.js +++ b/tests/unit/api/apiUtils/coldStorage.js @@ -4,7 +4,7 @@ const { errors } = require('arsenal'); const { startRestore, validatePutVersionId, - verifyColdObjectAvailable + verifyColdObjectAvailable, } = require('../../../../lib/api/apiUtils/object/coldStorage'); const { DummyRequestLogger } = require('../../helpers'); const { ObjectMD, ObjectMDArchive } = require('arsenal/build/lib/models'); @@ -13,9 +13,7 @@ const { scaledMsPerDay } = config.getTimeOptions(); const log = new DummyRequestLogger(); const oneDay = 24 * 60 * 60 * 1000; -const { - LOCATION_NAME_DMF, -} = require('../../../constants'); +const { LOCATION_NAME_DMF } = require('../../../constants'); describe('cold storage', () => { describe('validatePutVersionId', () => { @@ -74,29 +72,36 @@ describe('cold storage', () => { }, expectedRes: undefined, }, - ].forEach(testCase => it(testCase.description, () => { - const res = validatePutVersionId(testCase.objMD, testCase.versionId, log); - assert.deepStrictEqual(res, testCase.expectedRes); - })); + ].forEach(testCase => + it(testCase.description, () => { + const res = validatePutVersionId(testCase.objMD, testCase.versionId, log); + assert.deepStrictEqual(res, testCase.expectedRes); + }), + ); }); describe('verifyColdObjectAvailable', () => { [ { description: 'should return error if object is in a cold location', - objectMd: new ObjectMD() - .setArchive(new ObjectMDArchive({ + objectMd: new ObjectMD().setArchive( + new ObjectMDArchive({ archiveId: '97a71dfe-49c1-4cca-840a-69199e0b0322', - archiveVersion: 5577006791947779 - })) + archiveVersion: 5577006791947779, + }), + ), }, { description: 'should return error if object is restoring', - objectMd: new ObjectMD() - .setArchive(new ObjectMDArchive({ - archiveId: '97a71dfe-49c1-4cca-840a-69199e0b0322', - archiveVersion: 5577006791947779, - }, Date.now())) + objectMd: new ObjectMD().setArchive( + new ObjectMDArchive( + { + archiveId: '97a71dfe-49c1-4cca-840a-69199e0b0322', + archiveVersion: 5577006791947779, + }, + Date.now(), + ), + ), }, ].forEach(params => { it(`${params.description}`, () => { @@ -118,16 +123,18 @@ describe('cold storage', () => { }); it('should return null if object is restored', () => { - const objectMd = new ObjectMD().setArchive(new ObjectMDArchive( - { - archiveId: '97a71dfe-49c1-4cca-840a-69199e0b0322', - archiveVersion: 5577006791947779, - }, - /*restoreRequestedAt*/ new Date(0), - /*restoreRequestedDays*/ 5, - /*restoreCompletedAt*/ new Date(1000), - /*restoreWillExpireAt*/ new Date(1000 + 5 * oneDay), - )); + const objectMd = new ObjectMD().setArchive( + new ObjectMDArchive( + { + archiveId: '97a71dfe-49c1-4cca-840a-69199e0b0322', + archiveVersion: 5577006791947779, + }, + /*restoreRequestedAt*/ new Date(0), + /*restoreRequestedDays*/ 5, + /*restoreCompletedAt*/ new Date(1000), + /*restoreWillExpireAt*/ new Date(1000 + 5 * oneDay), + ), + ); const err = verifyColdObjectAvailable(objectMd.getValue()); assert.ifError(err); }); @@ -144,16 +151,19 @@ describe('cold storage', () => { }); it('should fail when object is being restored', done => { - const objectMd = new ObjectMD().setDataStoreName( - LOCATION_NAME_DMF - ).setArchive(new ObjectMDArchive( - { - archiveId: '97a71dfe-49c1-4cca-840a-69199e0b0322', - archiveVersion: 5577006791947779, - }, - /*restoreRequestedAt*/ new Date(0), - /*restoreRequestedDays*/ 5, - )).getValue(); + const objectMd = new ObjectMD() + .setDataStoreName(LOCATION_NAME_DMF) + .setArchive( + new ObjectMDArchive( + { + archiveId: '97a71dfe-49c1-4cca-840a-69199e0b0322', + archiveVersion: 5577006791947779, + }, + /*restoreRequestedAt*/ new Date(0), + /*restoreRequestedDays*/ 5, + ), + ) + .getValue(); startRestore(objectMd, { days: 5 }, log, err => { assert.deepStrictEqual(err, errors.RestoreAlreadyInProgress); @@ -162,18 +172,21 @@ describe('cold storage', () => { }); it('should fail when restored object is expired', done => { - const objectMd = new ObjectMD().setDataStoreName( - LOCATION_NAME_DMF - ).setArchive(new ObjectMDArchive( - { - archiveId: '97a71dfe-49c1-4cca-840a-69199e0b0322', - archiveVersion: 5577006791947779, - }, - /*restoreRequestedAt*/ new Date(0), - /*restoreRequestedDays*/ 5, - /*restoreCompletedAt*/ new Date(Date.now() - 6 * oneDay), - /*restoreWillExpireAt*/ new Date(Date.now() - 1 * oneDay), - )).getValue(); + const objectMd = new ObjectMD() + .setDataStoreName(LOCATION_NAME_DMF) + .setArchive( + new ObjectMDArchive( + { + archiveId: '97a71dfe-49c1-4cca-840a-69199e0b0322', + archiveVersion: 5577006791947779, + }, + /*restoreRequestedAt*/ new Date(0), + /*restoreRequestedDays*/ 5, + /*restoreCompletedAt*/ new Date(Date.now() - 6 * oneDay), + /*restoreWillExpireAt*/ new Date(Date.now() - 1 * oneDay), + ), + ) + .getValue(); startRestore(objectMd, { days: 5 }, log, err => { assert.deepStrictEqual(err, errors.InvalidObjectState); @@ -182,12 +195,15 @@ describe('cold storage', () => { }); it('should succeed for cold object', done => { - const objectMd = new ObjectMD().setDataStoreName( - LOCATION_NAME_DMF - ).setArchive(new ObjectMDArchive({ - archiveId: '97a71dfe-49c1-4cca-840a-69199e0b0322', - archiveVersion: 5577006791947779, - })).getValue(); + const objectMd = new ObjectMD() + .setDataStoreName(LOCATION_NAME_DMF) + .setArchive( + new ObjectMDArchive({ + archiveId: '97a71dfe-49c1-4cca-840a-69199e0b0322', + archiveVersion: 5577006791947779, + }), + ) + .getValue(); const t = new Date(); startRestore(objectMd, { days: 7 }, log, (err, isObjectAlreadyRestored) => { @@ -207,22 +223,26 @@ describe('cold storage', () => { }); it('should succeed for restored object', done => { - const objectMd = new ObjectMD().setDataStoreName( - LOCATION_NAME_DMF - ).setArchive(new ObjectMDArchive( - { - archiveId: '97a71dfe-49c1-4cca-840a-69199e0b0322', - archiveVersion: 5577006791947779, - }, - /*restoreRequestedAt*/ new Date(0), - /*restoreRequestedDays*/ 2, - /*restoreCompletedAt*/ new Date(Date.now() - 1 * oneDay), - /*restoreWillExpireAt*/ new Date(Date.now() + 1 * oneDay), - )).setAmzRestore({ - 'ongoing-request': false, - 'expiry-date': new Date(Date.now() + 1 * oneDay), - 'content-md5': '12345' - }).getValue(); + const objectMd = new ObjectMD() + .setDataStoreName(LOCATION_NAME_DMF) + .setArchive( + new ObjectMDArchive( + { + archiveId: '97a71dfe-49c1-4cca-840a-69199e0b0322', + archiveVersion: 5577006791947779, + }, + /*restoreRequestedAt*/ new Date(0), + /*restoreRequestedDays*/ 2, + /*restoreCompletedAt*/ new Date(Date.now() - 1 * oneDay), + /*restoreWillExpireAt*/ new Date(Date.now() + 1 * oneDay), + ), + ) + .setAmzRestore({ + 'ongoing-request': false, + 'expiry-date': new Date(Date.now() + 1 * oneDay), + 'content-md5': '12345', + }) + .getValue(); const restoreCompletedAt = objectMd.archive.restoreCompletedAt; const t = new Date(); @@ -236,12 +256,14 @@ describe('cold storage', () => { assert.ok(objectMd.archive.restoreRequestedAt.getTime() <= new Date()); assert.strictEqual(objectMd.archive.restoreCompletedAt, restoreCompletedAt); - assert.strictEqual(objectMd.archive.restoreWillExpireAt.getTime(), - objectMd.archive.restoreRequestedAt.getTime() + (5 * scaledMsPerDay)); + assert.strictEqual( + objectMd.archive.restoreWillExpireAt.getTime(), + objectMd.archive.restoreRequestedAt.getTime() + 5 * scaledMsPerDay, + ); assert.deepEqual(objectMd['x-amz-restore'], { 'ongoing-request': false, 'expiry-date': objectMd.archive.restoreWillExpireAt, - 'content-md5': '12345' + 'content-md5': '12345', }); done(); @@ -249,9 +271,7 @@ describe('cold storage', () => { }); it('should fail if _updateRestoreInfo fails', done => { - const objectMd = new ObjectMD().setDataStoreName( - LOCATION_NAME_DMF - ).setArchive(false).getValue(); + const objectMd = new ObjectMD().setDataStoreName(LOCATION_NAME_DMF).setArchive(false).getValue(); startRestore(objectMd, { days: 7 }, log, err => { assert.deepStrictEqual(err, errors.InternalError); diff --git a/tests/unit/api/apiUtils/expirationHeaders.js b/tests/unit/api/apiUtils/expirationHeaders.js index 41fa1af48c..07824e4854 100644 --- a/tests/unit/api/apiUtils/expirationHeaders.js +++ b/tests/unit/api/apiUtils/expirationHeaders.js @@ -1,25 +1,19 @@ const assert = require('assert'); const { LifecycleDateTime } = require('arsenal').s3middleware.lifecycleHelpers; - -const { - generateExpirationHeaders, -} = require('../../../../lib/api/apiUtils/object/expirationHeaders'); +const { generateExpirationHeaders } = require('../../../../lib/api/apiUtils/object/expirationHeaders'); const datetime = new LifecycleDateTime(); const objectDate = 'Fri, 21 Dec 2012 00:00:00 GMT'; const expectedDaysExpiryDate = 'Sat, 22 Dec 2012 00:00:00 GMT'; const expectedDateExpiryDate = 'Mon, 24 Dec 2012 00:00:00 GMT'; - const lifecycleExpirationDays = { rules: [ { ruleID: 'test-days', ruleStatus: 'Enabled', - actions: [ - { actionName: 'Expiration', days: 1 }, - ], + actions: [{ actionName: 'Expiration', days: 1 }], prefix: '', }, ], @@ -31,13 +25,9 @@ const lifecycleExpirationTags = { ruleID: 'test-tags', ruleStatus: 'Enabled', filters: { - tags: [ - { key: 'key1', val: 'val1' }, - ], + tags: [{ key: 'key1', val: 'val1' }], }, - actions: [ - { actionName: 'Expiration', days: 1 }, - ], + actions: [{ actionName: 'Expiration', days: 1 }], }, ], }; @@ -47,9 +37,7 @@ const lifecycleExpirationDate = { { ruleID: 'test-date', ruleStatus: 'Enabled', - actions: [ - { actionName: 'Expiration', date: 'Mon, 24 Dec 2012 00:00:00 GMT' }, - ], + actions: [{ actionName: 'Expiration', date: 'Mon, 24 Dec 2012 00:00:00 GMT' }], prefix: '', }, ], @@ -60,9 +48,7 @@ const lifecycleExpirationMPU = { { ruleID: 'test-mpu', ruleStatus: 'Enabled', - actions: [ - { actionName: 'AbortIncompleteMultipartUpload', days: 1 }, - ], + actions: [{ actionName: 'AbortIncompleteMultipartUpload', days: 1 }], prefix: '', }, ], @@ -172,7 +158,9 @@ describe('generateExpirationHeaders', () => { ], ]; - tests.forEach(([msg, params, expected]) => it(msg, () => { - assert.deepStrictEqual(generateExpirationHeaders(params, datetime), expected); - })); + tests.forEach(([msg, params, expected]) => + it(msg, () => { + assert.deepStrictEqual(generateExpirationHeaders(params, datetime), expected); + }), + ); }); diff --git a/tests/unit/api/apiUtils/getNotificationConfiguration.js b/tests/unit/api/apiUtils/getNotificationConfiguration.js index d5a531421c..8d226edb2e 100644 --- a/tests/unit/api/apiUtils/getNotificationConfiguration.js +++ b/tests/unit/api/apiUtils/getNotificationConfiguration.js @@ -3,8 +3,7 @@ const sinon = require('sinon'); const { config } = require('../../../../lib/Config'); const errors = require('arsenal').errors; -const getNotificationConfiguration = - require('../../../../lib/api/apiUtils/bucket/getNotificationConfiguration'); +const getNotificationConfiguration = require('../../../../lib/api/apiUtils/bucket/getNotificationConfiguration'); const parsedXml = { NotificationConfiguration: { @@ -15,18 +14,18 @@ const parsedXml = { Queue: ['arn:scality:bucketnotif:::target1'], }, ], - } + }, }; const expectedConfig = { queueConfig: [ { - events: ['s3:ObjectCreated:*'], - queueArn: 'arn:scality:bucketnotif:::target1', - id: 'notification-id', - filterRules: undefined - } - ] + events: ['s3:ObjectCreated:*'], + queueArn: 'arn:scality:bucketnotif:::target1', + id: 'notification-id', + filterRules: undefined, + }, + ], }; const destination1 = [ @@ -34,7 +33,7 @@ const destination1 = [ resource: 'target1', type: 'dummy', host: 'localhost:6000', - } + }, ]; const destinations2 = [ @@ -42,7 +41,7 @@ const destinations2 = [ resource: 'target2', type: 'dummy', host: 'localhost:6000', - } + }, ]; describe('getNotificationConfiguration', () => { @@ -58,7 +57,7 @@ describe('getNotificationConfiguration', () => { it('should return empty notification configuration', done => { sinon.stub(config, 'bucketNotificationDestinations').value(destination1); const notifConfig = getNotificationConfiguration({ - NotificationConfiguration: {} + NotificationConfiguration: {}, }); assert.deepEqual(notifConfig, {}); return done(); @@ -76,10 +75,12 @@ describe('getNotificationConfiguration', () => { const notifConfig = getNotificationConfiguration(parsedXml); assert.deepEqual(notifConfig.error, errors.InvalidArgument); const invalidArguments = notifConfig.error.metadata.get('invalidArguments'); - assert.deepEqual(invalidArguments, [{ - ArgumentName: 'arn:scality:bucketnotif:::target1', - ArgumentValue: 'The destination queue does not exist', - }]); + assert.deepEqual(invalidArguments, [ + { + ArgumentName: 'arn:scality:bucketnotif:::target1', + ArgumentValue: 'The destination queue does not exist', + }, + ]); return done(); }); }); diff --git a/tests/unit/api/apiUtils/getReplicationInfo.js b/tests/unit/api/apiUtils/getReplicationInfo.js index d8bec4c1e4..9134905bb4 100644 --- a/tests/unit/api/apiUtils/getReplicationInfo.js +++ b/tests/unit/api/apiUtils/getReplicationInfo.js @@ -2,15 +2,25 @@ const assert = require('assert'); const BucketInfo = require('arsenal').models.BucketInfo; const AuthInfo = require('arsenal').auth.AuthInfo; -const getReplicationInfo = - require('../../../../lib/api/apiUtils/object/getReplicationInfo'); +const getReplicationInfo = require('../../../../lib/api/apiUtils/object/getReplicationInfo'); function _getObjectReplicationInfo(s3config, replicationConfig) { const bucketInfo = new BucketInfo( - 'testbucket', 'someCanonicalId', 'accountDisplayName', + 'testbucket', + 'someCanonicalId', + 'accountDisplayName', new Date().toJSON(), - null, null, null, null, null, null, null, null, null, - replicationConfig); + null, + null, + null, + null, + null, + null, + null, + null, + null, + replicationConfig, + ); return getReplicationInfo(s3config, 'fookey', bucketInfo, true, 123, null, null); } @@ -36,39 +46,46 @@ const TEST_CONFIG = { azureStorageAccountName: 'fakeaccountname', azureStorageAccessKey: 'Fake00Key001', bucketMatch: true, - azureContainerName: 's3test' - } + azureContainerName: 's3test', + }, }, }, - replicationEndpoints: [{ - site: 'zenko', - servers: ['127.0.0.1:8000'], - default: true, - }, { - site: 'us-east-2', - type: 'aws_s3', - }], + replicationEndpoints: [ + { + site: 'zenko', + servers: ['127.0.0.1:8000'], + default: true, + }, + { + site: 'us-east-2', + type: 'aws_s3', + }, + ], }; describe('getReplicationInfo helper', () => { it('should get replication info when rules are enabled', () => { const replicationConfig = { role: 'arn:aws:iam::root:role/s3-replication-role', - rules: [{ - prefix: '', - enabled: true, - storageClass: 'awsbackend', - }], + rules: [ + { + prefix: '', + enabled: true, + storageClass: 'awsbackend', + }, + ], destination: 'tosomewhere', }; const replicationInfo = _getObjectReplicationInfo(TEST_CONFIG, replicationConfig); assert.deepStrictEqual(replicationInfo, { status: 'PENDING', - backends: [{ - site: 'awsbackend', - status: 'PENDING', - dataStoreVersionId: '', - }], + backends: [ + { + site: 'awsbackend', + status: 'PENDING', + dataStoreVersionId: '', + }, + ], content: ['METADATA'], destination: 'tosomewhere', storageClass: 'awsbackend', @@ -81,11 +98,13 @@ describe('getReplicationInfo helper', () => { it('should not get replication info when rules are disabled', () => { const replicationConfig = { role: 'arn:aws:iam::root:role/s3-replication-role', - rules: [{ - prefix: '', - enabled: false, - storageClass: 'awsbackend', - }], + rules: [ + { + prefix: '', + enabled: false, + storageClass: 'awsbackend', + }, + ], destination: 'tosomewhere', }; const replicationInfo = _getObjectReplicationInfo(TEST_CONFIG, replicationConfig); @@ -95,21 +114,25 @@ describe('getReplicationInfo helper', () => { it('should get replication info with single cloud target', () => { const replicationConfig = { role: 'arn:aws:iam::root:role/s3-replication-role', - rules: [{ - prefix: '', - enabled: true, - storageClass: 'awsbackend', - }], + rules: [ + { + prefix: '', + enabled: true, + storageClass: 'awsbackend', + }, + ], destination: 'tosomewhere', }; const replicationInfo = _getObjectReplicationInfo(TEST_CONFIG, replicationConfig); assert.deepStrictEqual(replicationInfo, { status: 'PENDING', - backends: [{ - site: 'awsbackend', - status: 'PENDING', - dataStoreVersionId: '', - }], + backends: [ + { + site: 'awsbackend', + status: 'PENDING', + dataStoreVersionId: '', + }, + ], content: ['METADATA'], destination: 'tosomewhere', storageClass: 'awsbackend', @@ -122,25 +145,30 @@ describe('getReplicationInfo helper', () => { it('should get replication info with multiple cloud targets', () => { const replicationConfig = { role: 'arn:aws:iam::root:role/s3-replication-role', - rules: [{ - prefix: '', - enabled: true, - storageClass: 'awsbackend,azurebackend', - }], + rules: [ + { + prefix: '', + enabled: true, + storageClass: 'awsbackend,azurebackend', + }, + ], destination: 'tosomewhere', }; const replicationInfo = _getObjectReplicationInfo(TEST_CONFIG, replicationConfig); assert.deepStrictEqual(replicationInfo, { status: 'PENDING', - backends: [{ - site: 'awsbackend', - status: 'PENDING', - dataStoreVersionId: '', - }, { - site: 'azurebackend', - status: 'PENDING', - dataStoreVersionId: '', - }], + backends: [ + { + site: 'awsbackend', + status: 'PENDING', + dataStoreVersionId: '', + }, + { + site: 'azurebackend', + status: 'PENDING', + dataStoreVersionId: '', + }, + ], content: ['METADATA'], destination: 'tosomewhere', storageClass: 'awsbackend,azurebackend', @@ -150,30 +178,34 @@ describe('getReplicationInfo helper', () => { }); }); - it('should get replication info with multiple cloud targets and ' + - 'preferred read location', () => { + it('should get replication info with multiple cloud targets and ' + 'preferred read location', () => { const replicationConfig = { role: 'arn:aws:iam::root:role/s3-replication-role', - rules: [{ - prefix: '', - enabled: true, - storageClass: 'awsbackend:preferred_read,azurebackend', - }], + rules: [ + { + prefix: '', + enabled: true, + storageClass: 'awsbackend:preferred_read,azurebackend', + }, + ], destination: 'tosomewhere', preferredReadLocation: 'awsbackend', }; const replicationInfo = _getObjectReplicationInfo(TEST_CONFIG, replicationConfig); assert.deepStrictEqual(replicationInfo, { status: 'PENDING', - backends: [{ - site: 'awsbackend', - status: 'PENDING', - dataStoreVersionId: '', - }, { - site: 'azurebackend', - status: 'PENDING', - dataStoreVersionId: '', - }], + backends: [ + { + site: 'awsbackend', + status: 'PENDING', + dataStoreVersionId: '', + }, + { + site: 'azurebackend', + status: 'PENDING', + dataStoreVersionId: '', + }, + ], content: ['METADATA'], destination: 'tosomewhere', storageClass: 'awsbackend:preferred_read,azurebackend', @@ -183,61 +215,84 @@ describe('getReplicationInfo helper', () => { }); }); - it('should not get replication info when service account type ' + - 'cannot trigger replication', () => { + it('should not get replication info when service account type ' + 'cannot trigger replication', () => { const replicationConfig = { role: 'arn:aws:iam::root:role/s3-replication-role', - rules: [{ - prefix: '', - enabled: true, - storageClass: 'awsbackend', - }], + rules: [ + { + prefix: '', + enabled: true, + storageClass: 'awsbackend', + }, + ], destination: 'tosomewhere', }; const bucketInfo = new BucketInfo( - 'testbucket', 'abcdef/lifecycle', 'Lifecycle Service Account', + 'testbucket', + 'abcdef/lifecycle', + 'Lifecycle Service Account', new Date().toJSON(), - null, null, null, null, null, null, null, null, null, - replicationConfig); + null, + null, + null, + null, + null, + null, + null, + null, + null, + replicationConfig, + ); const authInfo = new AuthInfo({ canonicalID: 'abcdef/lifecycle', accountDisplayName: 'Lifecycle Service Account', }); - const replicationInfo = getReplicationInfo(TEST_CONFIG, - 'fookey', bucketInfo, true, 123, null, null, authInfo); + const replicationInfo = getReplicationInfo(TEST_CONFIG, 'fookey', bucketInfo, true, 123, null, null, authInfo); assert.deepStrictEqual(replicationInfo, undefined); }); - it('should get replication info when service account type can ' + - 'trigger replication', () => { + it('should get replication info when service account type can ' + 'trigger replication', () => { const replicationConfig = { role: 'arn:aws:iam::root:role/s3-replication-role', - rules: [{ - prefix: '', - enabled: true, - storageClass: 'awsbackend', - }], + rules: [ + { + prefix: '', + enabled: true, + storageClass: 'awsbackend', + }, + ], destination: 'tosomewhere', }; const bucketInfo = new BucketInfo( - 'testbucket', 'abcdef/md-ingestion', + 'testbucket', + 'abcdef/md-ingestion', 'Metadata Ingestion Service Account', new Date().toJSON(), - null, null, null, null, null, null, null, null, null, - replicationConfig); + null, + null, + null, + null, + null, + null, + null, + null, + null, + replicationConfig, + ); const authInfo = new AuthInfo({ canonicalID: 'abcdef/md-ingestion', accountDisplayName: 'Metadata Ingestion Service Account', }); - const replicationInfo = getReplicationInfo(TEST_CONFIG, - 'fookey', bucketInfo, true, 123, null, null, authInfo); + const replicationInfo = getReplicationInfo(TEST_CONFIG, 'fookey', bucketInfo, true, 123, null, null, authInfo); assert.deepStrictEqual(replicationInfo, { status: 'PENDING', - backends: [{ - site: 'awsbackend', - status: 'PENDING', - dataStoreVersionId: '', - }], + backends: [ + { + site: 'awsbackend', + status: 'PENDING', + dataStoreVersionId: '', + }, + ], content: ['METADATA'], destination: 'tosomewhere', storageClass: 'awsbackend', @@ -250,20 +305,24 @@ describe('getReplicationInfo helper', () => { it('should get replication info with default StorageClass when rules are enabled', () => { const replicationConfig = { role: 'arn:aws:iam::root:role/s3-replication-role-1,arn:aws:iam::root:role/s3-replication-role-2', - rules: [{ - prefix: '', - enabled: true, - }], + rules: [ + { + prefix: '', + enabled: true, + }, + ], destination: 'tosomewhere', }; const replicationInfo = _getObjectReplicationInfo(TEST_CONFIG, replicationConfig); assert.deepStrictEqual(replicationInfo, { status: 'PENDING', - backends: [{ - site: 'zenko', - status: 'PENDING', - dataStoreVersionId: '', - }], + backends: [ + { + site: 'zenko', + status: 'PENDING', + dataStoreVersionId: '', + }, + ], content: ['METADATA'], destination: 'tosomewhere', storageClass: 'zenko', @@ -276,26 +335,29 @@ describe('getReplicationInfo helper', () => { it('should return undefined with specified StorageClass mode if no replication endpoint is configured', () => { const replicationConfig = { role: 'arn:aws:iam::root:role/s3-replication-role', - rules: [{ - prefix: '', - enabled: true, - storageClass: 'awsbackend', - }], + rules: [ + { + prefix: '', + enabled: true, + storageClass: 'awsbackend', + }, + ], destination: 'tosomewhere', }; const configWithNoReplicationEndpoint = { locationConstraints: TEST_CONFIG.locationConstraints, replicationEndpoints: [], }; - const replicationInfo = _getObjectReplicationInfo(configWithNoReplicationEndpoint, - replicationConfig); + const replicationInfo = _getObjectReplicationInfo(configWithNoReplicationEndpoint, replicationConfig); assert.deepStrictEqual(replicationInfo, { status: 'PENDING', - backends: [{ - site: 'awsbackend', - status: 'PENDING', - dataStoreVersionId: '', - }], + backends: [ + { + site: 'awsbackend', + status: 'PENDING', + dataStoreVersionId: '', + }, + ], content: ['METADATA'], destination: 'tosomewhere', storageClass: 'awsbackend', @@ -308,18 +370,19 @@ describe('getReplicationInfo helper', () => { it('should return undefined with default StorageClass if no replication endpoint is configured', () => { const replicationConfig = { role: 'arn:aws:iam::root:role/s3-replication-role-1,arn:aws:iam::root:role/s3-replication-role-2', - rules: [{ - prefix: '', - enabled: true, - }], + rules: [ + { + prefix: '', + enabled: true, + }, + ], destination: 'tosomewhere', }; const configWithNoReplicationEndpoint = { locationConstraints: TEST_CONFIG.locationConstraints, replicationEndpoints: [], }; - const replicationInfo = _getObjectReplicationInfo(configWithNoReplicationEndpoint, - replicationConfig); + const replicationInfo = _getObjectReplicationInfo(configWithNoReplicationEndpoint, replicationConfig); assert.deepStrictEqual(replicationInfo, undefined); }); }); diff --git a/tests/unit/api/apiUtils/integrity/validateChecksums.js b/tests/unit/api/apiUtils/integrity/validateChecksums.js index b3f2d89b99..6d8eb945b0 100644 --- a/tests/unit/api/apiUtils/integrity/validateChecksums.js +++ b/tests/unit/api/apiUtils/integrity/validateChecksums.js @@ -2,8 +2,11 @@ const assert = require('assert'); const crypto = require('crypto'); const sinon = require('sinon'); -const { validateChecksumsNoChunking, ChecksumError, validateMethodChecksumNoChunking } = - require('../../../../../lib/api/apiUtils/integrity/validateChecksums'); +const { + validateChecksumsNoChunking, + ChecksumError, + validateMethodChecksumNoChunking, +} = require('../../../../../lib/api/apiUtils/integrity/validateChecksums'); const { errors: ArsenalErrors } = require('arsenal'); const { config } = require('../../../../../lib/Config'); @@ -13,7 +16,7 @@ describe('validateChecksumsNoChunking', () => { const body = 'Hello, World!'; const expectedMd5 = crypto.createHash('md5').update(body, 'utf8').digest('base64'); const headers = { - 'content-md5': expectedMd5 + 'content-md5': expectedMd5, }; const result = validateChecksumsNoChunking(headers, body); @@ -27,7 +30,7 @@ describe('validateChecksumsNoChunking', () => { const wrongMd5 = 'wrongchecksum123='; const expectedMd5 = crypto.createHash('md5').update(body, 'utf8').digest('base64'); const headers = { - 'content-md5': wrongMd5 + 'content-md5': wrongMd5, }; const result = validateChecksumsNoChunking(headers, body); @@ -55,12 +58,12 @@ describe('validateChecksumsNoChunking', () => { assert.strictEqual(result.error, ChecksumError.MissingChecksum); assert.strictEqual(result.details, null); }); - + it('should return MD5Mismatch error when content-md5 header is undefined', () => { const body = 'Hello, World!'; const headers = { 'content-type': 'application/json', - 'content-md5': undefined + 'content-md5': undefined, }; const calculatedMD5 = crypto.createHash('md5').update(body, 'utf8').digest('base64'); @@ -74,7 +77,7 @@ describe('validateChecksumsNoChunking', () => { const body = 'Hello, World!'; const headers = { 'content-type': 'application/json', - 'content-md5': null + 'content-md5': null, }; const calculatedMD5 = crypto.createHash('md5').update(body, 'utf8').digest('base64'); @@ -88,7 +91,7 @@ describe('validateChecksumsNoChunking', () => { const body = 'Hello, World!'; const headers = { 'content-type': 'application/json', - 'content-md5': '' + 'content-md5': '', }; const calculatedMD5 = crypto.createHash('md5').update(body, 'utf8').digest('base64'); @@ -103,10 +106,10 @@ describe('validateChecksumsNoChunking', () => { describe('validateMethodChecksumNoChunking', () => { let sandbox; let originalIntegrityChecks; - + const supportedMethods = [ 'bucketPutACL', - 'bucketPutCors', + 'bucketPutCors', 'bucketPutEncryption', 'bucketPutLifecycle', 'bucketPutNotification', @@ -119,7 +122,7 @@ describe('validateMethodChecksumNoChunking', () => { 'objectPutACL', 'objectPutLegalHold', 'objectPutTagging', - 'objectPutRetention' + 'objectPutRetention', ]; beforeEach(() => { @@ -136,19 +139,19 @@ describe('validateMethodChecksumNoChunking', () => { supportedMethods.forEach(method => { it(`should return BadDigest error for ${method} when checksum mismatch`, () => { config.integrityChecks[method] = true; - + const body = 'Hello, World!'; const wrongMd5 = 'wrongchecksum123='; const request = { apiMethod: method, headers: { - 'content-md5': wrongMd5 - } + 'content-md5': wrongMd5, + }, }; const log = { debug: sandbox.stub() }; const result = validateMethodChecksumNoChunking(request, body, log); - + assert.deepStrictEqual(result, ArsenalErrors.BadDigest, 'Expected BadDigest error'); assert(log.debug.calledOnce); }); @@ -159,16 +162,16 @@ describe('validateMethodChecksumNoChunking', () => { supportedMethods.forEach(method => { it(`should return null for ${method} when no checksum is provided`, () => { config.integrityChecks[method] = true; - + const body = 'Hello, World!'; const request = { apiMethod: method, - headers: {} + headers: {}, }; const log = { debug: sandbox.stub() }; const result = validateMethodChecksumNoChunking(request, body, log); - + assert.strictEqual(result, null); assert(log.debug.notCalled); }); @@ -179,19 +182,19 @@ describe('validateMethodChecksumNoChunking', () => { supportedMethods.forEach(method => { it(`should return null for ${method} when checksum matches`, () => { config.integrityChecks[method] = true; - + const body = 'Hello, World!'; const correctMd5 = crypto.createHash('md5').update(body, 'utf8').digest('base64'); const request = { apiMethod: method, headers: { - 'content-md5': correctMd5 - } + 'content-md5': correctMd5, + }, }; const log = { debug: sandbox.stub() }; const result = validateMethodChecksumNoChunking(request, body, log); - + assert.strictEqual(result, null); assert(log.debug.notCalled); }); @@ -202,19 +205,19 @@ describe('validateMethodChecksumNoChunking', () => { supportedMethods.forEach(method => { it(`should return null for ${method} when disabled, even with checksum mismatch`, () => { config.integrityChecks[method] = false; - + const body = 'Hello, World!'; const wrongMd5 = 'wrongchecksum123='; const request = { apiMethod: method, headers: { - 'content-md5': wrongMd5 - } + 'content-md5': wrongMd5, + }, }; const log = { debug: sandbox.stub() }; const result = validateMethodChecksumNoChunking(request, body, log); - + assert.strictEqual(result, null); assert(log.debug.notCalled); }); @@ -225,19 +228,19 @@ describe('validateMethodChecksumNoChunking', () => { it('should return null for unsupported method even when enabled in config', () => { const unsupportedMethod = 'someUnsupportedMethod'; config.integrityChecks[unsupportedMethod] = true; - + const body = 'Hello, World!'; const wrongMd5 = 'wrongchecksum123='; const request = { apiMethod: unsupportedMethod, headers: { - 'content-md5': wrongMd5 - } + 'content-md5': wrongMd5, + }, }; const log = { debug: sandbox.stub() }; const result = validateMethodChecksumNoChunking(request, body, log); - + assert.strictEqual(result, null); assert(log.debug.notCalled); }); @@ -248,13 +251,13 @@ describe('validateMethodChecksumNoChunking', () => { const body = 'Hello, World!'; const request = { headers: { - 'content-md5': 'wrongchecksum123=' - } + 'content-md5': 'wrongchecksum123=', + }, }; const log = { debug: sandbox.stub() }; const result = validateMethodChecksumNoChunking(request, body, log); - + assert.strictEqual(result, null); }); @@ -263,13 +266,13 @@ describe('validateMethodChecksumNoChunking', () => { const request = { apiMethod: 'nonExistentMethod', headers: { - 'content-md5': 'wrongchecksum123=' - } + 'content-md5': 'wrongchecksum123=', + }, }; const log = { debug: sandbox.stub() }; const result = validateMethodChecksumNoChunking(request, body, log); - + assert.strictEqual(result, null); }); }); diff --git a/tests/unit/api/apiUtils/locationKeysHaveChanged.js b/tests/unit/api/apiUtils/locationKeysHaveChanged.js index 39d16712e9..7c0575ed56 100644 --- a/tests/unit/api/apiUtils/locationKeysHaveChanged.js +++ b/tests/unit/api/apiUtils/locationKeysHaveChanged.js @@ -1,6 +1,5 @@ const assert = require('assert'); -const locationKeysHaveChanged = - require('../../../../lib/api/apiUtils/object/locationKeysHaveChanged'); +const locationKeysHaveChanged = require('../../../../lib/api/apiUtils/object/locationKeysHaveChanged'); describe('Check if location keys have changed between object locations', () => { it('should return true for no match ', () => { diff --git a/tests/unit/api/apiUtils/object/objectAttributes.js b/tests/unit/api/apiUtils/object/objectAttributes.js index fe4a277e12..9844e3cf55 100644 --- a/tests/unit/api/apiUtils/object/objectAttributes.js +++ b/tests/unit/api/apiUtils/object/objectAttributes.js @@ -1,7 +1,7 @@ const assert = require('assert'); const { parseAttributesHeaders, - buildAttributesXml + buildAttributesXml, } = require('../../../../../lib/api/apiUtils/object/objectAttributes'); const headerName = 'x-amz-object-attributes'; @@ -48,16 +48,15 @@ describe('parseAttributesHeaders', () => { }); }); - describe('buildXmlAttributes', () => { const objectMD = { 'content-md5': '16e37e19194511993498801d4692795f', 'content-length': 5000, 'x-amz-storage-class': 'STANDARD', - 'restoreStatus': { + restoreStatus: { inProgress: false, - expiryDate: 'Fri, 20 Feb 2026 12:00:00 GMT' - } + expiryDate: 'Fri, 20 Feb 2026 12:00:00 GMT', + }, }; const userMetadata = { diff --git a/tests/unit/api/apiUtils/objectLockHelpers.js b/tests/unit/api/apiUtils/objectLockHelpers.js index 285efc9c8f..9769fb8a5b 100644 --- a/tests/unit/api/apiUtils/objectLockHelpers.js +++ b/tests/unit/api/apiUtils/objectLockHelpers.js @@ -16,14 +16,54 @@ const mockOwnerDisplayName = 'accountDisplayName'; const mockCreationDate = new Date().toJSON(); const bucketInfo = new BucketInfo( - mockName, mockOwner, mockOwnerDisplayName, mockCreationDate, - null, null, null, null, null, null, null, null, null, null, - null, null, null, null, null, null, null, true); + mockName, + mockOwner, + mockOwnerDisplayName, + mockCreationDate, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + true, +); const objLockDisabledBucketInfo = new BucketInfo( - mockName, mockOwner, mockOwnerDisplayName, mockCreationDate, - null, null, null, null, null, null, null, null, null, null, - null, null, null, null, null, null, null, false); + mockName, + mockOwner, + mockOwnerDisplayName, + mockCreationDate, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + false, +); const log = new DummyRequestLogger(); @@ -33,13 +73,12 @@ describe('objectLockHelpers: validateHeaders', () => { 'x-amz-object-lock-retain-until-date': '2050-10-12', 'x-amz-object-lock-mode': 'COMPLIANCE', }; - const objectLockValidationError - = validateHeaders(objLockDisabledBucketInfo, headers, log); + const objectLockValidationError = validateHeaders(objLockDisabledBucketInfo, headers, log); const expectedError = errorInstances.InvalidRequest.customizeDescription( - 'Bucket is missing ObjectLockConfiguration'); + 'Bucket is missing ObjectLockConfiguration', + ); assert.strictEqual(objectLockValidationError.is.InvalidRequest, true); - assert.strictEqual(objectLockValidationError.description, - expectedError.description); + assert.strictEqual(objectLockValidationError.description, expectedError.description); }); it('should pass with valid retention headers', () => { @@ -47,8 +86,7 @@ describe('objectLockHelpers: validateHeaders', () => { 'x-amz-object-lock-retain-until-date': '2050-10-12', 'x-amz-object-lock-mode': 'COMPLIANCE', }; - const objectLockValidationError - = validateHeaders(bucketInfo, headers, log); + const objectLockValidationError = validateHeaders(bucketInfo, headers, log); assert.strictEqual(objectLockValidationError, null); }); @@ -56,8 +94,7 @@ describe('objectLockHelpers: validateHeaders', () => { const headers = { 'x-amz-object-lock-legal-hold': 'ON', }; - const objectLockValidationError - = validateHeaders(bucketInfo, headers, log); + const objectLockValidationError = validateHeaders(bucketInfo, headers, log); assert.strictEqual(objectLockValidationError, null); }); @@ -65,8 +102,7 @@ describe('objectLockHelpers: validateHeaders', () => { const headers = { 'x-amz-object-lock-legal-hold': 'OFF', }; - const objectLockValidationError - = validateHeaders(bucketInfo, headers, log); + const objectLockValidationError = validateHeaders(bucketInfo, headers, log); assert.strictEqual(objectLockValidationError, null); }); @@ -76,8 +112,7 @@ describe('objectLockHelpers: validateHeaders', () => { 'x-amz-object-lock-mode': 'GOVERNANCE', 'x-amz-object-lock-legal-hold': 'ON', }; - const objectLockValidationError - = validateHeaders(bucketInfo, headers, log); + const objectLockValidationError = validateHeaders(bucketInfo, headers, log); assert.strictEqual(objectLockValidationError, null); }); @@ -85,28 +120,24 @@ describe('objectLockHelpers: validateHeaders', () => { const headers = { 'x-amz-object-lock-retain-until-date': '2005-10-12', }; - const objectLockValidationError - = validateHeaders(bucketInfo, headers, log); + const objectLockValidationError = validateHeaders(bucketInfo, headers, log); const expectedError = errorInstances.InvalidArgument.customizeDescription( - 'x-amz-object-lock-retain-until-date and x-amz-object-lock-mode ' + - 'must both be supplied'); + 'x-amz-object-lock-retain-until-date and x-amz-object-lock-mode ' + 'must both be supplied', + ); assert.strictEqual(objectLockValidationError.is.InvalidArgument, true); - assert.strictEqual(objectLockValidationError.description, - expectedError.description); + assert.strictEqual(objectLockValidationError.description, expectedError.description); }); it('should fail with missing object-lock-retain-until-date header', () => { const headers = { 'x-amz-object-lock-mode': 'GOVERNANCE', }; - const objectLockValidationError - = validateHeaders(bucketInfo, headers, log); + const objectLockValidationError = validateHeaders(bucketInfo, headers, log); const expectedError = errorInstances.InvalidArgument.customizeDescription( - 'x-amz-object-lock-retain-until-date and x-amz-object-lock-mode ' + - 'must both be supplied'); + 'x-amz-object-lock-retain-until-date and x-amz-object-lock-mode ' + 'must both be supplied', + ); assert.strictEqual(objectLockValidationError.is.InvalidArgument, true); - assert.strictEqual(objectLockValidationError.description, - expectedError.description); + assert.strictEqual(objectLockValidationError.description, expectedError.description); }); it('should fail with past retention date header', () => { @@ -115,25 +146,23 @@ describe('objectLockHelpers: validateHeaders', () => { 'x-amz-object-lock-mode': 'COMPLIANCE', }; const expectedError = errorInstances.InvalidArgument.customizeDescription( - 'The retain until date must be in the future!'); - const objectLockValidationError - = validateHeaders(bucketInfo, headers, log); + 'The retain until date must be in the future!', + ); + const objectLockValidationError = validateHeaders(bucketInfo, headers, log); assert.strictEqual(objectLockValidationError.is.InvalidArgument, true); - assert.strictEqual(objectLockValidationError.description, - expectedError.description); + assert.strictEqual(objectLockValidationError.description, expectedError.description); }); it('should fail with invalid legal hold header', () => { const headers = { 'x-amz-object-lock-legal-hold': 'on', }; - const objectLockValidationError - = validateHeaders(bucketInfo, headers, log); + const objectLockValidationError = validateHeaders(bucketInfo, headers, log); const expectedError = errorInstances.InvalidArgument.customizeDescription( - 'Legal hold status must be one of "ON", "OFF"'); + 'Legal hold status must be one of "ON", "OFF"', + ); assert.strictEqual(objectLockValidationError.is.InvalidArgument, true); - assert.strictEqual(objectLockValidationError.description, - expectedError.description); + assert.strictEqual(objectLockValidationError.description, expectedError.description); }); it('should fail with invalid retention period header', () => { @@ -141,13 +170,10 @@ describe('objectLockHelpers: validateHeaders', () => { 'x-amz-object-lock-retain-until-date': '2050-10-12', 'x-amz-object-lock-mode': 'Governance', }; - const objectLockValidationError - = validateHeaders(bucketInfo, headers, log); - const expectedError = errorInstances.InvalidArgument.customizeDescription( - 'Unknown wormMode directive'); + const objectLockValidationError = validateHeaders(bucketInfo, headers, log); + const expectedError = errorInstances.InvalidArgument.customizeDescription('Unknown wormMode directive'); assert.strictEqual(objectLockValidationError.is.InvalidArgument, true); - assert.strictEqual(objectLockValidationError.description, - expectedError.description); + assert.strictEqual(objectLockValidationError.description, expectedError.description); }); }); @@ -158,11 +184,9 @@ describe('objectLockHelpers: calculateRetainUntilDate', () => { days: 90, }; const date = moment(); - const expectedRetainUntilDate - = date.add(mockConfigWithDays.days * 86400000, 'ms'); + const expectedRetainUntilDate = date.add(mockConfigWithDays.days * 86400000, 'ms'); const retainUntilDate = calculateRetainUntilDate(mockConfigWithDays); - assert.strictEqual(retainUntilDate.slice(0, 16), - expectedRetainUntilDate.toISOString().slice(0, 16)); + assert.strictEqual(retainUntilDate.slice(0, 16), expectedRetainUntilDate.toISOString().slice(0, 16)); }); it('should calculate retainUntilDate for config with years', () => { @@ -171,11 +195,9 @@ describe('objectLockHelpers: calculateRetainUntilDate', () => { years: 3, }; const date = moment(); - const expectedRetainUntilDate - = date.add(mockConfigWithYears.years * 365 * 86400000, 'ms'); + const expectedRetainUntilDate = date.add(mockConfigWithYears.years * 365 * 86400000, 'ms'); const retainUntilDate = calculateRetainUntilDate(mockConfigWithYears); - assert.strictEqual(retainUntilDate.slice(0, 16), - expectedRetainUntilDate.toISOString().slice(0, 16)); + assert.strictEqual(retainUntilDate.slice(0, 16), expectedRetainUntilDate.toISOString().slice(0, 16)); }); }); @@ -269,7 +291,6 @@ describe('objectLockHelpers: compareObjectLockInformation', () => { }); }); - const pastDate = moment().subtract(1, 'days'); const futureDate = moment().add(100, 'days'); @@ -608,46 +629,50 @@ describe('objectLockHelpers: ObjectLockInfo', () => { }); }); - describe('isExpired: ', () => isExpiredTestCases.forEach(testCase => { - const objLockInfo = new ObjectLockInfo({ date: testCase.date }); - it(testCase.desc, () => assert.strictEqual(objLockInfo.isExpired(), testCase.expected)); - })); - - describe('isLocked: ', () => isLockedTestCases.forEach(testCase => { - describe(`${testCase.desc}`, () => { - it(`should show policy as ${testCase.expected ? '' : 'not'} locked without legal hold`, () => { - const objLockInfo = new ObjectLockInfo(testCase.policy); - assert.strictEqual(objLockInfo.isLocked(), testCase.expected); - }); - - // legal hold should show as locked regardless of policy - it('should show policy as locked with legal hold', () => { - const policy = Object.assign({}, testCase.policy, { legalHold: true }); - const objLockInfo = new ObjectLockInfo(policy); - assert.strictEqual(objLockInfo.isLocked(), true); + describe('isExpired: ', () => + isExpiredTestCases.forEach(testCase => { + const objLockInfo = new ObjectLockInfo({ date: testCase.date }); + it(testCase.desc, () => assert.strictEqual(objLockInfo.isExpired(), testCase.expected)); + })); + + describe('isLocked: ', () => + isLockedTestCases.forEach(testCase => { + describe(`${testCase.desc}`, () => { + it(`should show policy as ${testCase.expected ? '' : 'not'} locked without legal hold`, () => { + const objLockInfo = new ObjectLockInfo(testCase.policy); + assert.strictEqual(objLockInfo.isLocked(), testCase.expected); + }); + + // legal hold should show as locked regardless of policy + it('should show policy as locked with legal hold', () => { + const policy = Object.assign({}, testCase.policy, { legalHold: true }); + const objLockInfo = new ObjectLockInfo(policy); + assert.strictEqual(objLockInfo.isLocked(), true); + }); }); - }); - })); + })); - describe('canModifyPolicy: ', () => policyChangeTestCases.forEach(testCase => { - describe(testCase.desc, () => { - const objLockInfo = new ObjectLockInfo(testCase.from); - it(`should ${testCase.allowed ? 'allow' : 'deny'} modifying the policy without bypass`, - () => assert.strictEqual(objLockInfo.canModifyPolicy(testCase.to), testCase.allowed)); + describe('canModifyPolicy: ', () => + policyChangeTestCases.forEach(testCase => { + describe(testCase.desc, () => { + const objLockInfo = new ObjectLockInfo(testCase.from); + it(`should ${testCase.allowed ? 'allow' : 'deny'} modifying the policy without bypass`, () => + assert.strictEqual(objLockInfo.canModifyPolicy(testCase.to), testCase.allowed)); - it(`should ${testCase.allowedWithBypass ? 'allow' : 'deny'} modifying the policy with bypass`, - () => assert.strictEqual(objLockInfo.canModifyPolicy(testCase.to, true), testCase.allowedWithBypass)); - }); - })); + it(`should ${testCase.allowedWithBypass ? 'allow' : 'deny'} modifying the policy with bypass`, () => + assert.strictEqual(objLockInfo.canModifyPolicy(testCase.to, true), testCase.allowedWithBypass)); + }); + })); - describe('canModifyObject: ', () => canModifyObjectTestCases.forEach(testCase => { - describe(testCase.desc, () => { - const objLockInfo = new ObjectLockInfo(testCase.policy); - it(`should ${testCase.allowed ? 'allow' : 'deny'} modifying object without bypass`, - () => assert.strictEqual(objLockInfo.canModifyObject(), testCase.allowed)); + describe('canModifyObject: ', () => + canModifyObjectTestCases.forEach(testCase => { + describe(testCase.desc, () => { + const objLockInfo = new ObjectLockInfo(testCase.policy); + it(`should ${testCase.allowed ? 'allow' : 'deny'} modifying object without bypass`, () => + assert.strictEqual(objLockInfo.canModifyObject(), testCase.allowed)); - it(`should ${testCase.allowedWithBypass ? 'allow' : 'deny'} modifying object with bypass`, - () => assert.strictEqual(objLockInfo.canModifyObject(true), testCase.allowedWithBypass)); - }); - })); + it(`should ${testCase.allowedWithBypass ? 'allow' : 'deny'} modifying object with bypass`, () => + assert.strictEqual(objLockInfo.canModifyObject(true), testCase.allowedWithBypass)); + }); + })); }); diff --git a/tests/unit/api/apiUtils/permissionChecks.js b/tests/unit/api/apiUtils/permissionChecks.js index 0843682b3c..15d8427832 100644 --- a/tests/unit/api/apiUtils/permissionChecks.js +++ b/tests/unit/api/apiUtils/permissionChecks.js @@ -1,8 +1,13 @@ const assert = require('assert'); -const { isLifecycleSession, checkBucketPolicyResult, checkBucketPolicy, - isBucketAuthorized, isObjAuthorized, evaluateBucketPolicyWithIAM } = - require('../../../../lib/api/apiUtils/authorization/permissionChecks.js'); +const { + isLifecycleSession, + checkBucketPolicyResult, + checkBucketPolicy, + isBucketAuthorized, + isObjAuthorized, + evaluateBucketPolicyWithIAM, +} = require('../../../../lib/api/apiUtils/authorization/permissionChecks.js'); const { DummyRequestLogger } = require('../../helpers'); const stubLog = new DummyRequestLogger(); @@ -85,12 +90,8 @@ describe('checkBucketPolicy Principal logic', () => { Principal: { AWS: 'arn:aws:iam::123456789012:root', }, - Action: [ - 's3:*', - ], - Resource: [ - 'arn:aws:s3:::amzn-s3-demo-bucket', - ], + Action: ['s3:*'], + Resource: ['arn:aws:s3:::amzn-s3-demo-bucket'], }, ], }, @@ -105,7 +106,8 @@ describe('checkBucketPolicy Principal logic', () => { expectedResult: checkBucketPolicyResult.ALLOW, }, { - description: 'bucket owner and requester share the same account, principal account ID, Allow policy should return ALLOW', + description: + 'bucket owner and requester share the same account, principal account ID, Allow policy should return ALLOW', policy: { Statement: [ { @@ -114,12 +116,8 @@ describe('checkBucketPolicy Principal logic', () => { Principal: { AWS: '123456789012', }, - Action: [ - 's3:*', - ], - Resource: [ - 'arn:aws:s3:::amzn-s3-demo-bucket', - ], + Action: ['s3:*'], + Resource: ['arn:aws:s3:::amzn-s3-demo-bucket'], }, ], }, @@ -143,12 +141,8 @@ describe('checkBucketPolicy Principal logic', () => { Principal: { AWS: 'arn:aws:iam::123456789012:root', }, - Action: [ - 's3:*', - ], - Resource: [ - 'arn:aws:s3:::amzn-s3-demo-bucket', - ], + Action: ['s3:*'], + Resource: ['arn:aws:s3:::amzn-s3-demo-bucket'], }, ], }, @@ -163,7 +157,8 @@ describe('checkBucketPolicy Principal logic', () => { expectedResult: checkBucketPolicyResult.EXPLICIT_DENY, }, { - description: 'bucket owner and requester don\'t share the same account, Allow policy should return CROSS ACCOUNT', + description: + "bucket owner and requester don't share the same account, Allow policy should return CROSS ACCOUNT", policy: { Statement: [ { @@ -172,12 +167,8 @@ describe('checkBucketPolicy Principal logic', () => { Principal: { AWS: 'arn:aws:iam::123456789012:root', }, - Action: [ - 's3:*', - ], - Resource: [ - 'arn:aws:s3:::amzn-s3-demo-bucket', - ], + Action: ['s3:*'], + Resource: ['arn:aws:s3:::amzn-s3-demo-bucket'], }, ], }, @@ -192,7 +183,8 @@ describe('checkBucketPolicy Principal logic', () => { expectedResult: checkBucketPolicyResult.CROSS_ACCOUNT_ALLOW, }, { - description: 'bucket owner and requester don\'t share the same account, principal account ID, Allow policy should return CROSS ACCOUNT', + description: + "bucket owner and requester don't share the same account, principal account ID, Allow policy should return CROSS ACCOUNT", policy: { Statement: [ { @@ -201,12 +193,8 @@ describe('checkBucketPolicy Principal logic', () => { Principal: { AWS: '123456789012', }, - Action: [ - 's3:*', - ], - Resource: [ - 'arn:aws:s3:::amzn-s3-demo-bucket', - ], + Action: ['s3:*'], + Resource: ['arn:aws:s3:::amzn-s3-demo-bucket'], }, ], }, @@ -221,7 +209,7 @@ describe('checkBucketPolicy Principal logic', () => { expectedResult: checkBucketPolicyResult.CROSS_ACCOUNT_ALLOW, }, { - description: 'bucket owner and requester don\'t share the same account, Deny policy should return DENY', + description: "bucket owner and requester don't share the same account, Deny policy should return DENY", policy: { Statement: [ { @@ -230,12 +218,8 @@ describe('checkBucketPolicy Principal logic', () => { Principal: { AWS: 'arn:aws:iam::123456789012:root', }, - Action: [ - 's3:*', - ], - Resource: [ - 'arn:aws:s3:::amzn-s3-demo-bucket', - ], + Action: ['s3:*'], + Resource: ['arn:aws:s3:::amzn-s3-demo-bucket'], }, ], }, @@ -250,7 +234,8 @@ describe('checkBucketPolicy Principal logic', () => { expectedResult: checkBucketPolicyResult.EXPLICIT_DENY, }, { - description: 'bucket owner and requester don\'t share the same account, requester is root, Allow policy should return ALLOW', + description: + "bucket owner and requester don't share the same account, requester is root, Allow policy should return ALLOW", policy: { Statement: [ { @@ -259,12 +244,8 @@ describe('checkBucketPolicy Principal logic', () => { Principal: { AWS: 'arn:aws:iam::123456789012:root', }, - Action: [ - 's3:*', - ], - Resource: [ - 'arn:aws:s3:::amzn-s3-demo-bucket', - ], + Action: ['s3:*'], + Resource: ['arn:aws:s3:::amzn-s3-demo-bucket'], }, ], }, @@ -279,7 +260,8 @@ describe('checkBucketPolicy Principal logic', () => { expectedResult: checkBucketPolicyResult.ALLOW, }, { - description: 'bucket owner and requester don\'t share the same account, requester is root, Deny policy should return DENY', + description: + "bucket owner and requester don't share the same account, requester is root, Deny policy should return DENY", policy: { Statement: [ { @@ -288,12 +270,8 @@ describe('checkBucketPolicy Principal logic', () => { Principal: { AWS: 'arn:aws:iam::123456789012:root', }, - Action: [ - 's3:*', - ], - Resource: [ - 'arn:aws:s3:::amzn-s3-demo-bucket', - ], + Action: ['s3:*'], + Resource: ['arn:aws:s3:::amzn-s3-demo-bucket'], }, ], }, @@ -308,7 +286,8 @@ describe('checkBucketPolicy Principal logic', () => { expectedResult: checkBucketPolicyResult.EXPLICIT_DENY, }, { - description: 'bucket owner and requester don\'t share the same account, requester and principal are users, Allow policy should return CROSS_ACCOUNT', + description: + "bucket owner and requester don't share the same account, requester and principal are users, Allow policy should return CROSS_ACCOUNT", policy: { Statement: [ { @@ -317,12 +296,8 @@ describe('checkBucketPolicy Principal logic', () => { Principal: { AWS: 'arn:aws:iam::123456789012:user/testuser', }, - Action: [ - 's3:*', - ], - Resource: [ - 'arn:aws:s3:::amzn-s3-demo-bucket', - ], + Action: ['s3:*'], + Resource: ['arn:aws:s3:::amzn-s3-demo-bucket'], }, ], }, @@ -337,7 +312,8 @@ describe('checkBucketPolicy Principal logic', () => { expectedResult: checkBucketPolicyResult.CROSS_ACCOUNT_ALLOW, }, { - description: 'bucket owner and requester don\'t share the same account, requester and principal are users, Deny policy should return EXPLICIT_DENY', + description: + "bucket owner and requester don't share the same account, requester and principal are users, Deny policy should return EXPLICIT_DENY", policy: { Statement: [ { @@ -346,12 +322,8 @@ describe('checkBucketPolicy Principal logic', () => { Principal: { AWS: 'arn:aws:iam::123456789012:user/testuser', }, - Action: [ - 's3:*', - ], - Resource: [ - 'arn:aws:s3:::amzn-s3-demo-bucket', - ], + Action: ['s3:*'], + Resource: ['arn:aws:s3:::amzn-s3-demo-bucket'], }, ], }, @@ -366,7 +338,8 @@ describe('checkBucketPolicy Principal logic', () => { expectedResult: checkBucketPolicyResult.EXPLICIT_DENY, }, { - description: 'bucket owner and requester don\'t share the same account, wildcard "*" principal, Allow policy should return CROSS_ACCOUNT_ALLOW', + description: + 'bucket owner and requester don\'t share the same account, wildcard "*" principal, Allow policy should return CROSS_ACCOUNT_ALLOW', policy: { Statement: [ { @@ -375,12 +348,8 @@ describe('checkBucketPolicy Principal logic', () => { Principal: { AWS: '*', }, - Action: [ - 's3:*', - ], - Resource: [ - 'arn:aws:s3:::amzn-s3-demo-bucket', - ], + Action: ['s3:*'], + Resource: ['arn:aws:s3:::amzn-s3-demo-bucket'], }, ], }, @@ -395,7 +364,8 @@ describe('checkBucketPolicy Principal logic', () => { expectedResult: checkBucketPolicyResult.CROSS_ACCOUNT_ALLOW, }, { - description: 'bucket owner and requester share the same account, wildcard "*" principal, Allow policy should return ALLOW', + description: + 'bucket owner and requester share the same account, wildcard "*" principal, Allow policy should return ALLOW', policy: { Statement: [ { @@ -404,12 +374,8 @@ describe('checkBucketPolicy Principal logic', () => { Principal: { AWS: '*', }, - Action: [ - 's3:*', - ], - Resource: [ - 'arn:aws:s3:::amzn-s3-demo-bucket', - ], + Action: ['s3:*'], + Resource: ['arn:aws:s3:::amzn-s3-demo-bucket'], }, ], }, @@ -424,19 +390,16 @@ describe('checkBucketPolicy Principal logic', () => { expectedResult: checkBucketPolicyResult.ALLOW, }, { - description: 'bucket owner and requester share the same account, wildcard "*" principal, string typeof principal , Allow policy should return ALLOW', + description: + 'bucket owner and requester share the same account, wildcard "*" principal, string typeof principal , Allow policy should return ALLOW', policy: { Statement: [ { Sid: 'Example permissions', Effect: 'Allow', Principal: '*', - Action: [ - 's3:*', - ], - Resource: [ - 'arn:aws:s3:::amzn-s3-demo-bucket', - ], + Action: ['s3:*'], + Resource: ['arn:aws:s3:::amzn-s3-demo-bucket'], }, ], }, @@ -451,19 +414,16 @@ describe('checkBucketPolicy Principal logic', () => { expectedResult: checkBucketPolicyResult.ALLOW, }, { - description: 'bucket owner and requester don\'t share the same account, wildcard "*" principal, string typeof principal, Allow policy should return CROSS_ACCOUNT_ALLOW', + description: + 'bucket owner and requester don\'t share the same account, wildcard "*" principal, string typeof principal, Allow policy should return CROSS_ACCOUNT_ALLOW', policy: { Statement: [ { Sid: 'Example permissions', Effect: 'Allow', Principal: '*', - Action: [ - 's3:*', - ], - Resource: [ - 'arn:aws:s3:::amzn-s3-demo-bucket', - ], + Action: ['s3:*'], + Resource: ['arn:aws:s3:::amzn-s3-demo-bucket'], }, ], }, @@ -478,7 +438,8 @@ describe('checkBucketPolicy Principal logic', () => { expectedResult: checkBucketPolicyResult.CROSS_ACCOUNT_ALLOW, }, { - description: 'bucket owner and requester don\'t share the same account, no bucket policy for user, canonical user principal, Allow policy should return DEFAULT_DENY', + description: + "bucket owner and requester don't share the same account, no bucket policy for user, canonical user principal, Allow policy should return DEFAULT_DENY", policy: { Statement: [ { @@ -487,12 +448,8 @@ describe('checkBucketPolicy Principal logic', () => { Principal: { CanonicalUser: 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx', }, - Action: [ - 's3:*', - ], - Resource: [ - 'arn:aws:s3:::amzn-s3-demo-bucket', - ], + Action: ['s3:*'], + Resource: ['arn:aws:s3:::amzn-s3-demo-bucket'], }, ], }, @@ -507,7 +464,8 @@ describe('checkBucketPolicy Principal logic', () => { expectedResult: checkBucketPolicyResult.DEFAULT_DENY, }, { - description: 'bucket owner and requester don\'t share the same account, canonical user principal, Allow policy should return CROSS_ACCOUNT_ALLOW', + description: + "bucket owner and requester don't share the same account, canonical user principal, Allow policy should return CROSS_ACCOUNT_ALLOW", policy: { Statement: [ { @@ -516,12 +474,8 @@ describe('checkBucketPolicy Principal logic', () => { Principal: { CanonicalUser: '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be', }, - Action: [ - 's3:*', - ], - Resource: [ - 'arn:aws:s3:::amzn-s3-demo-bucket', - ], + Action: ['s3:*'], + Resource: ['arn:aws:s3:::amzn-s3-demo-bucket'], }, ], }, @@ -545,12 +499,8 @@ describe('checkBucketPolicy Principal logic', () => { Principal: { CanonicalUser: '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be', }, - Action: [ - 's3:*', - ], - Resource: [ - 'arn:aws:s3:::amzn-s3-demo-bucket', - ], + Action: ['s3:*'], + Resource: ['arn:aws:s3:::amzn-s3-demo-bucket'], }, { Sid: 'Example permissions', @@ -558,12 +508,8 @@ describe('checkBucketPolicy Principal logic', () => { Principal: { CanonicalUser: '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be', }, - Action: [ - 's3:*', - ], - Resource: [ - 'arn:aws:s3:::amzn-s3-demo-bucket', - ], + Action: ['s3:*'], + Resource: ['arn:aws:s3:::amzn-s3-demo-bucket'], }, ], }, @@ -587,12 +533,8 @@ describe('checkBucketPolicy Principal logic', () => { Principal: { CanonicalUser: '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be', }, - Action: [ - 's3:*', - ], - Resource: [ - 'arn:aws:s3:::amzn-s3-demo-bucket', - ], + Action: ['s3:*'], + Resource: ['arn:aws:s3:::amzn-s3-demo-bucket'], }, { Sid: 'Example permissions', @@ -600,12 +542,8 @@ describe('checkBucketPolicy Principal logic', () => { Principal: { CanonicalUser: '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be', }, - Action: [ - 's3:*', - ], - Resource: [ - 'arn:aws:s3:::amzn-s3-demo-bucket', - ], + Action: ['s3:*'], + Resource: ['arn:aws:s3:::amzn-s3-demo-bucket'], }, ], }, @@ -627,12 +565,8 @@ describe('checkBucketPolicy Principal logic', () => { Sid: 'Example permissions', Effect: 'Allow', Principal: '*', - Action: [ - 's3:*', - ], - Resource: [ - 'arn:aws:s3:::amzn-s3-demo-bucket', - ], + Action: ['s3:*'], + Resource: ['arn:aws:s3:::amzn-s3-demo-bucket'], }, ], }, @@ -654,12 +588,8 @@ describe('checkBucketPolicy Principal logic', () => { Sid: 'Example permissions', Effect: 'Allow', Principal: { AWS: '*' }, - Action: [ - 's3:*', - ], - Resource: [ - 'arn:aws:s3:::amzn-s3-demo-bucket', - ], + Action: ['s3:*'], + Resource: ['arn:aws:s3:::amzn-s3-demo-bucket'], }, ], }, @@ -681,12 +611,8 @@ describe('checkBucketPolicy Principal logic', () => { Sid: 'Example permissions', Effect: 'Allow', Principal: { CanonicalUser: '*' }, - Action: [ - 's3:*', - ], - Resource: [ - 'arn:aws:s3:::amzn-s3-demo-bucket', - ], + Action: ['s3:*'], + Resource: ['arn:aws:s3:::amzn-s3-demo-bucket'], }, ], }, @@ -704,8 +630,16 @@ describe('checkBucketPolicy Principal logic', () => { tests.forEach(t => { it(t.description, () => { - const res = checkBucketPolicy(t.policy, t.requestType, t.canonicalID, t.arn, t.bucketOwner, - t.log, t.request, t.actionImplicitDenies); + const res = checkBucketPolicy( + t.policy, + t.requestType, + t.canonicalID, + t.arn, + t.bucketOwner, + t.log, + t.request, + t.actionImplicitDenies, + ); assert.equal(res, t.expectedResult); }); }); @@ -743,8 +677,7 @@ describe('aclRequired field in isBucketAuthorized', () => { const request = makeRequest(); const bucket = makeBucket(ownerCanonicalId); const authInfo = makeAuthInfo(ownerCanonicalId, 'arn:aws:iam::123456789012:/owner/'); - isBucketAuthorized(bucket, 'bucketGet', ownerCanonicalId, authInfo, stubLog, - request, { bucketGet: false }); + isBucketAuthorized(bucket, 'bucketGet', ownerCanonicalId, authInfo, stubLog, request, { bucketGet: false }); assert.strictEqual(request.serverAccessLog.aclRequired, undefined); }); @@ -752,8 +685,7 @@ describe('aclRequired field in isBucketAuthorized', () => { const request = makeRequest(); const bucket = makeBucket(ownerCanonicalId, { READ: [otherCanonicalId] }); const authInfo = makeAuthInfo(otherCanonicalId, 'arn:aws:iam::999999999999:user/other'); - isBucketAuthorized(bucket, 'bucketGet', otherCanonicalId, authInfo, stubLog, - request, { bucketGet: false }); + isBucketAuthorized(bucket, 'bucketGet', otherCanonicalId, authInfo, stubLog, request, { bucketGet: false }); assert.strictEqual(request.serverAccessLog.aclRequired, 'Yes'); }); @@ -762,34 +694,44 @@ describe('aclRequired field in isBucketAuthorized', () => { // Use an IAM user in the bucket owner's account so principal match is OK (not CROSS_ACCOUNT) // and the owner early-return doesn't fire (requester is IAM user, not account root) const iamUserCanonicalId = ownerCanonicalId; - const bucket = makeBucket(ownerCanonicalId, {}, { - Statement: [{ - Effect: 'Allow', - Principal: { AWS: 'arn:aws:iam::123456789012:user/iamuser' }, - Action: ['s3:ListBucket'], - Resource: ['arn:aws:s3:::test-bucket'], - }], - }); + const bucket = makeBucket( + ownerCanonicalId, + {}, + { + Statement: [ + { + Effect: 'Allow', + Principal: { AWS: 'arn:aws:iam::123456789012:user/iamuser' }, + Action: ['s3:ListBucket'], + Resource: ['arn:aws:s3:::test-bucket'], + }, + ], + }, + ); const authInfo = makeAuthInfo(iamUserCanonicalId, 'arn:aws:iam::123456789012:user/iamuser', true); - isBucketAuthorized(bucket, 'bucketGet', iamUserCanonicalId, authInfo, stubLog, - request, { bucketGet: false }); + isBucketAuthorized(bucket, 'bucketGet', iamUserCanonicalId, authInfo, stubLog, request, { bucketGet: false }); assert.strictEqual(request.serverAccessLog.aclRequired, undefined); }); it('should not set aclRequired when bucket policy explicitly denies', () => { const request = makeRequest(); const iamUserCanonicalId = ownerCanonicalId; - const bucket = makeBucket(ownerCanonicalId, { READ: [iamUserCanonicalId] }, { - Statement: [{ - Effect: 'Deny', - Principal: { AWS: 'arn:aws:iam::123456789012:user/iamuser' }, - Action: ['s3:ListBucket'], - Resource: ['arn:aws:s3:::test-bucket'], - }], - }); + const bucket = makeBucket( + ownerCanonicalId, + { READ: [iamUserCanonicalId] }, + { + Statement: [ + { + Effect: 'Deny', + Principal: { AWS: 'arn:aws:iam::123456789012:user/iamuser' }, + Action: ['s3:ListBucket'], + Resource: ['arn:aws:s3:::test-bucket'], + }, + ], + }, + ); const authInfo = makeAuthInfo(iamUserCanonicalId, 'arn:aws:iam::123456789012:user/iamuser', true); - isBucketAuthorized(bucket, 'bucketGet', iamUserCanonicalId, authInfo, stubLog, - request, { bucketGet: false }); + isBucketAuthorized(bucket, 'bucketGet', iamUserCanonicalId, authInfo, stubLog, request, { bucketGet: false }); assert.strictEqual(request.serverAccessLog.aclRequired, undefined); }); @@ -798,17 +740,22 @@ describe('aclRequired field in isBucketAuthorized', () => { // Policy grants PutObject to a different principal — nothing matches // the bucketGet request from otherCanonicalId, so checkBucketPolicy // returns DEFAULT_DENY and falls back to ACL evaluation. - const bucket = makeBucket(ownerCanonicalId, { READ: [otherCanonicalId] }, { - Statement: [{ - Effect: 'Allow', - Principal: { AWS: 'arn:aws:iam::111111111111:root' }, - Action: ['s3:PutObject'], - Resource: ['arn:aws:s3:::test-bucket/*'], - }], - }); + const bucket = makeBucket( + ownerCanonicalId, + { READ: [otherCanonicalId] }, + { + Statement: [ + { + Effect: 'Allow', + Principal: { AWS: 'arn:aws:iam::111111111111:root' }, + Action: ['s3:PutObject'], + Resource: ['arn:aws:s3:::test-bucket/*'], + }, + ], + }, + ); const authInfo = makeAuthInfo(otherCanonicalId, 'arn:aws:iam::999999999999:user/other'); - isBucketAuthorized(bucket, 'bucketGet', otherCanonicalId, authInfo, stubLog, - request, { bucketGet: false }); + isBucketAuthorized(bucket, 'bucketGet', otherCanonicalId, authInfo, stubLog, request, { bucketGet: false }); assert.strictEqual(request.serverAccessLog.aclRequired, 'Yes'); }); @@ -817,8 +764,7 @@ describe('aclRequired field in isBucketAuthorized', () => { const bucket = makeBucket(ownerCanonicalId, { READ: [otherCanonicalId] }); const authInfo = makeAuthInfo(otherCanonicalId, 'arn:aws:iam::999999999999:user/other'); assert.doesNotThrow(() => { - isBucketAuthorized(bucket, 'bucketGet', otherCanonicalId, authInfo, stubLog, - request, { bucketGet: false }); + isBucketAuthorized(bucket, 'bucketGet', otherCanonicalId, authInfo, stubLog, request, { bucketGet: false }); }); }); }); @@ -871,8 +817,9 @@ describe('aclRequired field in isObjAuthorized', () => { const bucket = makeBucket(ownerCanonicalId); const objectMD = makeObjectMD(otherCanonicalId); const authInfo = makeAuthInfo(otherCanonicalId, 'arn:aws:iam::999999999999:/account/'); - isObjAuthorized(bucket, objectMD, 'objectGet', otherCanonicalId, authInfo, stubLog, - request, { objectGet: false }); + isObjAuthorized(bucket, objectMD, 'objectGet', otherCanonicalId, authInfo, stubLog, request, { + objectGet: false, + }); assert.strictEqual(request.serverAccessLog.aclRequired, undefined); }); @@ -882,8 +829,9 @@ describe('aclRequired field in isObjAuthorized', () => { const objectMD = makeObjectMD(ownerCanonicalId); objectMD.acl.READ = [otherCanonicalId]; const authInfo = makeAuthInfo(otherCanonicalId, 'arn:aws:iam::999999999999:user/other'); - isObjAuthorized(bucket, objectMD, 'objectGet', otherCanonicalId, authInfo, stubLog, - request, { objectGet: false }); + isObjAuthorized(bucket, objectMD, 'objectGet', otherCanonicalId, authInfo, stubLog, request, { + objectGet: false, + }); assert.strictEqual(request.serverAccessLog.aclRequired, 'Yes'); }); }); @@ -916,8 +864,15 @@ describe('aclRequired field in evaluateBucketPolicyWithIAM', () => { const request = { serverAccessLog: {} }; const bucket = makeBucket(ownerCanonicalId); const authInfo = makeAuthInfo(otherCanonicalId, 'arn:aws:iam::999999999999:user/other'); - evaluateBucketPolicyWithIAM(bucket, 'objectDelete', otherCanonicalId, authInfo, - { objectDelete: false }, stubLog, request); + evaluateBucketPolicyWithIAM( + bucket, + 'objectDelete', + otherCanonicalId, + authInfo, + { objectDelete: false }, + stubLog, + request, + ); assert.strictEqual(request.serverAccessLog.aclRequired, undefined); }); }); diff --git a/tests/unit/api/apiUtils/quotas/quotaUtils.js b/tests/unit/api/apiUtils/quotas/quotaUtils.js index 67b4750c43..3ebf496c62 100644 --- a/tests/unit/api/apiUtils/quotas/quotaUtils.js +++ b/tests/unit/api/apiUtils/quotas/quotaUtils.js @@ -75,15 +75,13 @@ describe('validateQuotas (buckets)', () => { validateQuotas(request, mockBucket, {}, ['objectPut', 'getObject'], 'objectPut', 1, false, mockLog, err => { assert.ifError(err); assert.strictEqual(QuotaService._getLatestMetricsCallback.calledOnce, true); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith( - 'bucket', - 'bucketName_1640995200000', - null, - { + assert.strictEqual( + QuotaService._getLatestMetricsCallback.calledWith('bucket', 'bucketName_1640995200000', null, { action: 'objectPut', inflight: 1, - } - ), true); + }), + true, + ); done(); }); }); @@ -102,15 +100,13 @@ describe('validateQuotas (buckets)', () => { assert.strictEqual(err.is.QuotaExceeded, true); assert.strictEqual(QuotaService._getLatestMetricsCallback.callCount, 1); assert.strictEqual(request.finalizerHooks.length, 1); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith( - 'bucket', - 'bucketName_1640995200000', - null, - { + assert.strictEqual( + QuotaService._getLatestMetricsCallback.calledWith('bucket', 'bucketName_1640995200000', null, { action: 'objectPut', inflight: 1, - } - ), true); + }), + true, + ); done(); }); }); @@ -128,15 +124,13 @@ describe('validateQuotas (buckets)', () => { validateQuotas(request, mockBucket, {}, ['objectDelete'], 'objectDelete', 0, false, mockLog, err => { assert.ifError(err); assert.strictEqual(QuotaService._getLatestMetricsCallback.calledOnce, true); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith( - 'bucket', - 'bucketName_1640995200000', - null, - { + assert.strictEqual( + QuotaService._getLatestMetricsCallback.calledWith('bucket', 'bucketName_1640995200000', null, { action: 'objectDelete', inflight: 0, - } - ), true); + }), + true, + ); done(); }); }); @@ -154,15 +148,13 @@ describe('validateQuotas (buckets)', () => { validateQuotas(request, mockBucket, {}, ['objectDelete'], 'objectDelete', -50, false, mockLog, err => { assert.ifError(err); assert.strictEqual(QuotaService._getLatestMetricsCallback.calledOnce, true); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith( - 'bucket', - 'bucketName_1640995200000', - null, - { + assert.strictEqual( + QuotaService._getLatestMetricsCallback.calledWith('bucket', 'bucketName_1640995200000', null, { action: 'objectDelete', inflight: -50, - } - ), true); + }), + true, + ); done(); }); }); @@ -180,15 +172,13 @@ describe('validateQuotas (buckets)', () => { validateQuotas(request, mockBucket, {}, ['objectDelete'], 'objectDeleteVersion', -50, false, mockLog, err => { assert.ifError(err); assert.strictEqual(QuotaService._getLatestMetricsCallback.calledOnce, true); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith( - 'bucket', - 'bucketName_1640995200000', - null, - { + assert.strictEqual( + QuotaService._getLatestMetricsCallback.calledWith('bucket', 'bucketName_1640995200000', null, { action: 'objectDelete', inflight: -50, - } - ), true); + }), + true, + ); done(); }); }); @@ -206,15 +196,13 @@ describe('validateQuotas (buckets)', () => { validateQuotas(request, mockBucket, {}, ['objectDelete'], 'objectDelete', -5000, false, mockLog, err => { assert.ifError(err); assert.strictEqual(QuotaService._getLatestMetricsCallback.calledOnce, true); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith( - 'bucket', - 'bucketName_1640995200000', - null, - { + assert.strictEqual( + QuotaService._getLatestMetricsCallback.calledWith('bucket', 'bucketName_1640995200000', null, { action: 'objectDelete', inflight: -5000, - } - ), true); + }), + true, + ); done(); }); }); @@ -229,21 +217,28 @@ describe('validateQuotas (buckets)', () => { QuotaService._getLatestMetricsCallback.yields(null, result1); QuotaService._getLatestMetricsCallback.onCall(1).yields(null, result2); - validateQuotas(request, mockBucket, {}, ['objectRestore', 'objectPut'], 'objectRestore', - true, false, mockLog, err => { + validateQuotas( + request, + mockBucket, + {}, + ['objectRestore', 'objectPut'], + 'objectRestore', + true, + false, + mockLog, + err => { assert.ifError(err); assert.strictEqual(QuotaService._getLatestMetricsCallback.calledTwice, true); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith( - 'bucket', - 'bucketName_1640995200000', - null, - { + assert.strictEqual( + QuotaService._getLatestMetricsCallback.calledWith('bucket', 'bucketName_1640995200000', null, { action: 'objectRestore', inflight: true, - } - ), true); + }), + true, + ); done(); - }); + }, + ); }); it('should not include the inflights in the request if they are disabled', done => { @@ -257,21 +252,28 @@ describe('validateQuotas (buckets)', () => { QuotaService._getLatestMetricsCallback.yields(null, result1); QuotaService._getLatestMetricsCallback.onCall(1).yields(null, result2); - validateQuotas(request, mockBucket, {}, ['objectRestore', 'objectPut'], 'objectRestore', - true, false, mockLog, err => { + validateQuotas( + request, + mockBucket, + {}, + ['objectRestore', 'objectPut'], + 'objectRestore', + true, + false, + mockLog, + err => { assert.ifError(err); assert.strictEqual(QuotaService._getLatestMetricsCallback.calledTwice, true); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith( - 'bucket', - 'bucketName_1640995200000', - null, - { + assert.strictEqual( + QuotaService._getLatestMetricsCallback.calledWith('bucket', 'bucketName_1640995200000', null, { action: 'objectRestore', inflight: undefined, - } - ), true); - done(); - }); + }), + true, + ); + done(); + }, + ); }); it('should evaluate the quotas and not update the inflights when isStorageReserved is true', done => { @@ -284,21 +286,18 @@ describe('validateQuotas (buckets)', () => { QuotaService._getLatestMetricsCallback.yields(null, result1); QuotaService._getLatestMetricsCallback.onCall(1).yields(null, result2); - validateQuotas(request, mockBucket, {}, ['objectPut'], 'objectPut', - true, true, mockLog, err => { - assert.ifError(err); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledOnce, true); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith( - 'bucket', - 'bucketName_1640995200000', - null, - { - action: 'objectPut', - inflight: 0, - } - ), true); - done(); - }); + validateQuotas(request, mockBucket, {}, ['objectPut'], 'objectPut', true, true, mockLog, err => { + assert.ifError(err); + assert.strictEqual(QuotaService._getLatestMetricsCallback.calledOnce, true); + assert.strictEqual( + QuotaService._getLatestMetricsCallback.calledWith('bucket', 'bucketName_1640995200000', null, { + action: 'objectPut', + inflight: 0, + }), + true, + ); + done(); + }); }); it('should handle numbers above MAX_SAFE_INTEGER when quota is not exceeded', done => { @@ -308,23 +307,31 @@ describe('validateQuotas (buckets)', () => { }; QuotaService._getLatestMetricsCallback.yields(null, result1); - validateQuotas(request, { - ...mockBucket, - getQuota: () => 9007199254740993n, - }, {}, ['objectPut'], 'objectPut', 1, false, mockLog, err => { - assert.ifError(err); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledOnce, true); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith( - 'bucket', - 'bucketName_1640995200000', - null, - { - action: 'objectPut', - inflight: 1, - }, - ), true); - done(); - }); + validateQuotas( + request, + { + ...mockBucket, + getQuota: () => 9007199254740993n, + }, + {}, + ['objectPut'], + 'objectPut', + 1, + false, + mockLog, + err => { + assert.ifError(err); + assert.strictEqual(QuotaService._getLatestMetricsCallback.calledOnce, true); + assert.strictEqual( + QuotaService._getLatestMetricsCallback.calledWith('bucket', 'bucketName_1640995200000', null, { + action: 'objectPut', + inflight: 1, + }), + true, + ); + done(); + }, + ); }); it('should handle numbers above MAX_SAFE_INTEGER when quota is exceeded', done => { @@ -334,15 +341,25 @@ describe('validateQuotas (buckets)', () => { }; QuotaService._getLatestMetricsCallback.yields(null, result1); - validateQuotas(request, { - ...mockBucket, - getQuota: () => 9007199254740991n, - }, {}, ['objectPut'], 'objectPut', 1, false, mockLog, err => { - assert.strictEqual(err.is.QuotaExceeded, true); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledOnce, true); - assert.strictEqual(request.finalizerHooks.length, 1); - done(); - }); + validateQuotas( + request, + { + ...mockBucket, + getQuota: () => 9007199254740991n, + }, + {}, + ['objectPut'], + 'objectPut', + 1, + false, + mockLog, + err => { + assert.strictEqual(err.is.QuotaExceeded, true); + assert.strictEqual(QuotaService._getLatestMetricsCallback.calledOnce, true); + assert.strictEqual(request.finalizerHooks.length, 1); + done(); + }, + ); }); it('should handle numbers above MAX_SAFE_INTEGER with disabled inflights when quota is not exceeded', done => { @@ -353,23 +370,31 @@ describe('validateQuotas (buckets)', () => { }; QuotaService._getLatestMetricsCallback.yields(null, result1); - validateQuotas(request, { - ...mockBucket, - getQuota: () => 9007199254740993n, - }, {}, ['objectPut'], 'objectPut', 1, false, mockLog, err => { - assert.ifError(err); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledOnce, true); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith( - 'bucket', - 'bucketName_1640995200000', - null, - { - action: 'objectPut', - inflight: undefined, - }, - ), true); - done(); - }); + validateQuotas( + request, + { + ...mockBucket, + getQuota: () => 9007199254740993n, + }, + {}, + ['objectPut'], + 'objectPut', + 1, + false, + mockLog, + err => { + assert.ifError(err); + assert.strictEqual(QuotaService._getLatestMetricsCallback.calledOnce, true); + assert.strictEqual( + QuotaService._getLatestMetricsCallback.calledWith('bucket', 'bucketName_1640995200000', null, { + action: 'objectPut', + inflight: undefined, + }), + true, + ); + done(); + }, + ); }); }); @@ -398,37 +423,67 @@ describe('validateQuotas (with accounts)', () => { }); it('should return null if quota is <= 0', done => { - validateQuotas(request, mockBucketNoQuota, { - account: 'test_1', - quota: 0n, - }, [], '', false, false, mockLog, err => { - assert.ifError(err); - assert.strictEqual(QuotaService._getLatestMetricsCallback.called, false); - done(); - }); + validateQuotas( + request, + mockBucketNoQuota, + { + account: 'test_1', + quota: 0n, + }, + [], + '', + false, + false, + mockLog, + err => { + assert.ifError(err); + assert.strictEqual(QuotaService._getLatestMetricsCallback.called, false); + done(); + }, + ); }); it('should not return null if bucket quota is <= 0 but account quota is > 0', done => { - validateQuotas(request, mockBucketNoQuota, { - account: 'test_1', - quota: 1000n, - }, [], '', false, false, mockLog, err => { - assert.ifError(err); - assert.strictEqual(QuotaService._getLatestMetricsCallback.called, false); - done(); - }); + validateQuotas( + request, + mockBucketNoQuota, + { + account: 'test_1', + quota: 1000n, + }, + [], + '', + false, + false, + mockLog, + err => { + assert.ifError(err); + assert.strictEqual(QuotaService._getLatestMetricsCallback.called, false); + done(); + }, + ); }); it('should return null if scuba is disabled', done => { QuotaService.enabled = false; - validateQuotas(request, mockBucket, { - account: 'test_1', - quota: 1000n, - }, [], '', false, false, mockLog, err => { - assert.ifError(err); - assert.strictEqual(QuotaService._getLatestMetricsCallback.called, false); - done(); - }); + validateQuotas( + request, + mockBucket, + { + account: 'test_1', + quota: 1000n, + }, + [], + '', + false, + false, + mockLog, + err => { + assert.ifError(err); + assert.strictEqual(QuotaService._getLatestMetricsCallback.called, false); + done(); + }, + ); }); it('should return null if metrics retrieval fails', done => { @@ -436,23 +491,31 @@ describe('validateQuotas (with accounts)', () => { const error = new Error('Failed to get metrics'); QuotaService._getLatestMetricsCallback.yields(error); - validateQuotas(request, mockBucket, { - account: 'test_1', - quota: 1000n, - }, ['objectPut', 'getObject'], 'objectPut', 1, false, mockLog, err => { - assert.ifError(err); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledOnce, true); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith( - 'bucket', - 'bucketName_1640995200000', - null, - { - action: 'objectPut', - inflight: 1, - } - ), true); - done(); - }); + validateQuotas( + request, + mockBucket, + { + account: 'test_1', + quota: 1000n, + }, + ['objectPut', 'getObject'], + 'objectPut', + 1, + false, + mockLog, + err => { + assert.ifError(err); + assert.strictEqual(QuotaService._getLatestMetricsCallback.calledOnce, true); + assert.strictEqual( + QuotaService._getLatestMetricsCallback.calledWith('bucket', 'bucketName_1640995200000', null, { + action: 'objectPut', + inflight: 1, + }), + true, + ); + done(); + }, + ); }); it('should return errors.QuotaExceeded if quota is exceeded', done => { @@ -465,24 +528,32 @@ describe('validateQuotas (with accounts)', () => { QuotaService._getLatestMetricsCallback.yields(null, result1); QuotaService._getLatestMetricsCallback.onCall(1).yields(null, result2); - validateQuotas(request, mockBucketNoQuota, { - account: 'test_1', - quota: 100n, - }, ['objectPut', 'getObject'], 'objectPut', 1, false, mockLog, err => { - assert.strictEqual(err.is.QuotaExceeded, true); - assert.strictEqual(QuotaService._getLatestMetricsCallback.callCount, 1); - assert.strictEqual(request.finalizerHooks.length, 1); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith( - 'account', - 'test_1', - null, - { - action: 'objectPut', - inflight: 1, - } - ), true); - done(); - }); + validateQuotas( + request, + mockBucketNoQuota, + { + account: 'test_1', + quota: 100n, + }, + ['objectPut', 'getObject'], + 'objectPut', + 1, + false, + mockLog, + err => { + assert.strictEqual(err.is.QuotaExceeded, true); + assert.strictEqual(QuotaService._getLatestMetricsCallback.callCount, 1); + assert.strictEqual(request.finalizerHooks.length, 1); + assert.strictEqual( + QuotaService._getLatestMetricsCallback.calledWith('account', 'test_1', null, { + action: 'objectPut', + inflight: 1, + }), + true, + ); + done(); + }, + ); }); it('should not return QuotaExceeded if the quotas are exceeded but operation is a delete', done => { @@ -495,23 +566,31 @@ describe('validateQuotas (with accounts)', () => { QuotaService._getLatestMetricsCallback.yields(null, result1); QuotaService._getLatestMetricsCallback.onCall(1).yields(null, result2); - validateQuotas(request, mockBucketNoQuota, { - account: 'test_1', - quota: 1000n, - }, ['objectDelete'], 'objectDelete', -50, false, mockLog, err => { - assert.ifError(err); - assert.strictEqual(QuotaService._getLatestMetricsCallback.callCount, 1); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith( - 'account', - 'test_1', - null, - { - action: 'objectDelete', - inflight: -50, - } - ), true); - done(); - }); + validateQuotas( + request, + mockBucketNoQuota, + { + account: 'test_1', + quota: 1000n, + }, + ['objectDelete'], + 'objectDelete', + -50, + false, + mockLog, + err => { + assert.ifError(err); + assert.strictEqual(QuotaService._getLatestMetricsCallback.callCount, 1); + assert.strictEqual( + QuotaService._getLatestMetricsCallback.calledWith('account', 'test_1', null, { + action: 'objectDelete', + inflight: -50, + }), + true, + ); + done(); + }, + ); }); it('should decrease the inflights by deleting data, and go below 0 to unblock operations', done => { @@ -524,23 +603,31 @@ describe('validateQuotas (with accounts)', () => { QuotaService._getLatestMetricsCallback.yields(null, result1); QuotaService._getLatestMetricsCallback.onCall(1).yields(null, result2); - validateQuotas(request, mockBucketNoQuota, { - account: 'test_1', - quota: 1000n, - }, ['objectDelete'], 'objectDelete', -5000, false, mockLog, err => { - assert.ifError(err); - assert.strictEqual(QuotaService._getLatestMetricsCallback.callCount, 1); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith( - 'account', - 'test_1', - null, - { - action: 'objectDelete', - inflight: -5000, - } - ), true); - done(); - }); + validateQuotas( + request, + mockBucketNoQuota, + { + account: 'test_1', + quota: 1000n, + }, + ['objectDelete'], + 'objectDelete', + -5000, + false, + mockLog, + err => { + assert.ifError(err); + assert.strictEqual(QuotaService._getLatestMetricsCallback.callCount, 1); + assert.strictEqual( + QuotaService._getLatestMetricsCallback.calledWith('account', 'test_1', null, { + action: 'objectDelete', + inflight: -5000, + }), + true, + ); + done(); + }, + ); }); it('should return null if quota is not exceeded', done => { @@ -553,23 +640,31 @@ describe('validateQuotas (with accounts)', () => { QuotaService._getLatestMetricsCallback.yields(null, result1); QuotaService._getLatestMetricsCallback.onCall(1).yields(null, result2); - validateQuotas(request, mockBucket, { - account: 'test_1', - quota: 1000n, - }, ['objectRestore', 'objectPut'], 'objectRestore', true, false, mockLog, err => { - assert.ifError(err); - assert.strictEqual(QuotaService._getLatestMetricsCallback.callCount, 4); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith( - 'account', - 'test_1', - null, - { - action: 'objectRestore', - inflight: true, - } - ), true); - done(); - }); + validateQuotas( + request, + mockBucket, + { + account: 'test_1', + quota: 1000n, + }, + ['objectRestore', 'objectPut'], + 'objectRestore', + true, + false, + mockLog, + err => { + assert.ifError(err); + assert.strictEqual(QuotaService._getLatestMetricsCallback.callCount, 4); + assert.strictEqual( + QuotaService._getLatestMetricsCallback.calledWith('account', 'test_1', null, { + action: 'objectRestore', + inflight: true, + }), + true, + ); + done(); + }, + ); }); it('should return quota exceeded if account and bucket quotas are different', done => { @@ -582,15 +677,25 @@ describe('validateQuotas (with accounts)', () => { QuotaService._getLatestMetricsCallback.yields(null, result1); QuotaService._getLatestMetricsCallback.onCall(1).yields(null, result2); - validateQuotas(request, mockBucket, { - account: 'test_1', - quota: 1000n, - }, ['objectPut', 'getObject'], 'objectPut', 1, false, mockLog, err => { - assert.strictEqual(err.is.QuotaExceeded, true); - assert.strictEqual(QuotaService._getLatestMetricsCallback.callCount, 2); - assert.strictEqual(request.finalizerHooks.length, 1); - done(); - }); + validateQuotas( + request, + mockBucket, + { + account: 'test_1', + quota: 1000n, + }, + ['objectPut', 'getObject'], + 'objectPut', + 1, + false, + mockLog, + err => { + assert.strictEqual(err.is.QuotaExceeded, true); + assert.strictEqual(QuotaService._getLatestMetricsCallback.callCount, 2); + assert.strictEqual(request.finalizerHooks.length, 1); + done(); + }, + ); }); it('should update the request with one function per action to clear quota updates', done => { @@ -603,23 +708,31 @@ describe('validateQuotas (with accounts)', () => { QuotaService._getLatestMetricsCallback.yields(null, result1); QuotaService._getLatestMetricsCallback.onCall(1).yields(null, result2); - validateQuotas(request, mockBucket, { - account: 'test_1', - quota: 1000n, - }, ['objectRestore', 'objectPut'], 'objectRestore', true, false, mockLog, err => { - assert.ifError(err); - assert.strictEqual(QuotaService._getLatestMetricsCallback.callCount, 4); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith( - 'account', - 'test_1', - null, - { - action: 'objectRestore', - inflight: true, - } - ), true); - done(); - }); + validateQuotas( + request, + mockBucket, + { + account: 'test_1', + quota: 1000n, + }, + ['objectRestore', 'objectPut'], + 'objectRestore', + true, + false, + mockLog, + err => { + assert.ifError(err); + assert.strictEqual(QuotaService._getLatestMetricsCallback.callCount, 4); + assert.strictEqual( + QuotaService._getLatestMetricsCallback.calledWith('account', 'test_1', null, { + action: 'objectRestore', + inflight: true, + }), + true, + ); + done(); + }, + ); }); it('should evaluate the quotas and not update the inflights when isStorageReserved is true', done => { @@ -632,23 +745,31 @@ describe('validateQuotas (with accounts)', () => { QuotaService._getLatestMetricsCallback.yields(null, result1); QuotaService._getLatestMetricsCallback.onCall(1).yields(null, result2); - validateQuotas(request, mockBucket, { - account: 'test_1', - quota: 1000n, - }, ['objectPut'], 'objectPut', true, true, mockLog, err => { - assert.ifError(err); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledTwice, true); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith( - 'account', - 'test_1', - null, - { - action: 'objectPut', - inflight: 0, - } - ), true); - done(); - }); + validateQuotas( + request, + mockBucket, + { + account: 'test_1', + quota: 1000n, + }, + ['objectPut'], + 'objectPut', + true, + true, + mockLog, + err => { + assert.ifError(err); + assert.strictEqual(QuotaService._getLatestMetricsCallback.calledTwice, true); + assert.strictEqual( + QuotaService._getLatestMetricsCallback.calledWith('account', 'test_1', null, { + action: 'objectPut', + inflight: 0, + }), + true, + ); + done(); + }, + ); }); it('should handle account numbers above MAX_SAFE_INTEGER when quota is not exceeded', done => { @@ -658,23 +779,31 @@ describe('validateQuotas (with accounts)', () => { }; QuotaService._getLatestMetricsCallback.yields(null, result1); - validateQuotas(request, mockBucketNoQuota, { - account: 'test_1', - quota: 9007199254740993n, - }, ['objectPut'], 'objectPut', 1, false, mockLog, err => { - assert.ifError(err); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledOnce, true); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith( - 'account', - 'test_1', - null, - { - action: 'objectPut', - inflight: 1, - }, - ), true); - done(); - }); + validateQuotas( + request, + mockBucketNoQuota, + { + account: 'test_1', + quota: 9007199254740993n, + }, + ['objectPut'], + 'objectPut', + 1, + false, + mockLog, + err => { + assert.ifError(err); + assert.strictEqual(QuotaService._getLatestMetricsCallback.calledOnce, true); + assert.strictEqual( + QuotaService._getLatestMetricsCallback.calledWith('account', 'test_1', null, { + action: 'objectPut', + inflight: 1, + }), + true, + ); + done(); + }, + ); }); it('should handle account numbers above MAX_SAFE_INTEGER when quota is exceeded', done => { @@ -684,15 +813,25 @@ describe('validateQuotas (with accounts)', () => { }; QuotaService._getLatestMetricsCallback.yields(null, result1); - validateQuotas(request, mockBucketNoQuota, { - account: 'test_1', - quota: 9007199254740991n, - }, ['objectPut'], 'objectPut', 1, false, mockLog, err => { - assert.strictEqual(err.is.QuotaExceeded, true); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledOnce, true); - assert.strictEqual(request.finalizerHooks.length, 1); - done(); - }); + validateQuotas( + request, + mockBucketNoQuota, + { + account: 'test_1', + quota: 9007199254740991n, + }, + ['objectPut'], + 'objectPut', + 1, + false, + mockLog, + err => { + assert.strictEqual(err.is.QuotaExceeded, true); + assert.strictEqual(QuotaService._getLatestMetricsCallback.calledOnce, true); + assert.strictEqual(request.finalizerHooks.length, 1); + done(); + }, + ); }); it('should handle account numbers above MAX_SAFE_INTEGER with disabled inflights', done => { @@ -703,23 +842,31 @@ describe('validateQuotas (with accounts)', () => { }; QuotaService._getLatestMetricsCallback.yields(null, result1); - validateQuotas(request, mockBucketNoQuota, { - account: 'test_1', - quota: 9007199254740993n, - }, ['objectPut'], 'objectPut', 1, false, mockLog, err => { - assert.ifError(err); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledOnce, true); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith( - 'account', - 'test_1', - null, - { - action: 'objectPut', - inflight: undefined, - }, - ), true); - done(); - }); + validateQuotas( + request, + mockBucketNoQuota, + { + account: 'test_1', + quota: 9007199254740993n, + }, + ['objectPut'], + 'objectPut', + 1, + false, + mockLog, + err => { + assert.ifError(err); + assert.strictEqual(QuotaService._getLatestMetricsCallback.calledOnce, true); + assert.strictEqual( + QuotaService._getLatestMetricsCallback.calledWith('account', 'test_1', null, { + action: 'objectPut', + inflight: undefined, + }), + true, + ); + done(); + }, + ); }); }); @@ -746,7 +893,7 @@ describe('processBytesToWrite', () => { ...hotObject, dataStoreName: 'glacier', archive: { - archiveInfo: '{archiveID,archiveVersion}' + archiveInfo: '{archiveID,archiveVersion}', }, }; const restoringObject = { diff --git a/tests/unit/api/apiUtils/rateLimit/cache.js b/tests/unit/api/apiUtils/rateLimit/cache.js index cb3e705fa9..421fa6c9c0 100644 --- a/tests/unit/api/apiUtils/rateLimit/cache.js +++ b/tests/unit/api/apiUtils/rateLimit/cache.js @@ -29,13 +29,10 @@ describe('test limit config cache storage', () => { it('should add config to cache', () => { setCachedConfig(namespace.bucket, 'foo', 10, constants.rateLimitDefaultConfigCacheTTL); - assert.deepStrictEqual( - configCache.get(`${namespace.bucket}:foo`), - { - expiry: now + constants.rateLimitDefaultConfigCacheTTL, - value: 10, - } - ); + assert.deepStrictEqual(configCache.get(`${namespace.bucket}:foo`), { + expiry: now + constants.rateLimitDefaultConfigCacheTTL, + value: 10, + }); }); it('should get a non expired config', () => { diff --git a/tests/unit/api/apiUtils/rateLimit/cleanup.js b/tests/unit/api/apiUtils/rateLimit/cleanup.js index eab3e5901b..db2721c8a2 100644 --- a/tests/unit/api/apiUtils/rateLimit/cleanup.js +++ b/tests/unit/api/apiUtils/rateLimit/cleanup.js @@ -1,10 +1,7 @@ const assert = require('assert'); const sinon = require('sinon'); -const { - startCleanupJob, - stopCleanupJob, -} = require('../../../../../lib/api/apiUtils/rateLimit/cleanup'); +const { startCleanupJob, stopCleanupJob } = require('../../../../../lib/api/apiUtils/rateLimit/cleanup'); const constants = require('../../../../../constants'); describe('Rate limit cleanup job', () => { @@ -32,9 +29,11 @@ describe('Rate limit cleanup job', () => { startCleanupJob(mockLog, { skipUnref: true }); assert(mockLog.info.calledOnce); - assert(mockLog.info.calledWith('Starting rate limit cleanup job', { - interval: constants.rateLimitCleanupInterval, - })); + assert( + mockLog.info.calledWith('Starting rate limit cleanup job', { + interval: constants.rateLimitCleanupInterval, + }), + ); assert(setTimeoutSpy.calledOnce); assert.strictEqual(setTimeoutSpy.firstCall.args[1], constants.rateLimitCleanupInterval); }); diff --git a/tests/unit/api/apiUtils/tagConditionKeys.js b/tests/unit/api/apiUtils/tagConditionKeys.js index 3a86628a7d..31ce684f93 100644 --- a/tests/unit/api/apiUtils/tagConditionKeys.js +++ b/tests/unit/api/apiUtils/tagConditionKeys.js @@ -8,8 +8,11 @@ const { TaggingConfigTester, createRequestContext, } = require('../../helpers'); -const { tagConditionKeyAuth, updateRequestContextsWithTags, makeTagQuery } = - require('../../../../lib/api/apiUtils/authorization/tagConditionKeys'); +const { + tagConditionKeyAuth, + updateRequestContextsWithTags, + makeTagQuery, +} = require('../../../../lib/api/apiUtils/authorization/tagConditionKeys'); const { bucketPut } = require('../../../../lib/api/bucketPut'); const objectPut = require('../../../../lib/api/objectPut'); @@ -29,21 +32,22 @@ const bucketPutReq = { const taggingUtil = new TaggingConfigTester(); -const objectPutReq = new DummyRequest({ - bucketName, - namespace, - objectKey, - headers: { - 'host': `${bucketName}.s3.amazonaws.com`, - 'x-amz-tagging': makeTagQuery(taggingUtil.getTags()), +const objectPutReq = new DummyRequest( + { + bucketName, + namespace, + objectKey, + headers: { + host: `${bucketName}.s3.amazonaws.com`, + 'x-amz-tagging': makeTagQuery(taggingUtil.getTags()), + }, + url: `/${bucketName}/${objectKey}`, + calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', }, - url: `/${bucketName}/${objectKey}`, - calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', -}, postBody); + postBody, +); -const objectPutRequestContexts = [ - createRequestContext('objectPut', objectPutReq), -]; +const objectPutRequestContexts = [createRequestContext('objectPut', objectPutReq)]; const objectGetReq = { bucketName, @@ -87,8 +91,7 @@ describe('updateRequestContextsWithTags', () => { updateRequestContextsWithTags(objectPutReq, objectPutRequestContexts, 'objectPut', log, err => { assert.ifError(err); assert(objectPutRequestContexts[0].getNeedTagEval()); - assert.strictEqual(objectPutRequestContexts[0].getRequestObjTags(), - makeTagQuery(taggingUtil.getTags())); + assert.strictEqual(objectPutRequestContexts[0].getRequestObjTags(), makeTagQuery(taggingUtil.getTags())); assert.strictEqual(objectPutRequestContexts[0].getExistingObjTag(), null); done(); }); @@ -97,14 +100,12 @@ describe('updateRequestContextsWithTags', () => { it('should update multiple request contexts with existing object tags', done => { objectPut(authInfo, objectPutReq, 'foobar', log, err => { assert.ifError(err); - updateRequestContextsWithTags(objectGetReq, objectGetRequestContexts, 'objectGet', log, - err => { + updateRequestContextsWithTags(objectGetReq, objectGetRequestContexts, 'objectGet', log, err => { assert.ifError(err); // FIXME introduced by CLDSRV-256, this syntax should be allowed by the linter for (const requestContext of objectGetRequestContexts) { assert(requestContext.getNeedTagEval()); - assert.strictEqual(requestContext.getExistingObjTag(), - makeTagQuery(taggingUtil.getTags())); + assert.strictEqual(requestContext.getExistingObjTag(), makeTagQuery(taggingUtil.getTags())); assert.strictEqual(requestContext.getRequestObjTags(), null); } done(); diff --git a/tests/unit/api/apiUtils/validateChecksumHeaders.js b/tests/unit/api/apiUtils/validateChecksumHeaders.js index 6b1f7dbbf6..b1abd6709c 100644 --- a/tests/unit/api/apiUtils/validateChecksumHeaders.js +++ b/tests/unit/api/apiUtils/validateChecksumHeaders.js @@ -55,7 +55,6 @@ unsupportedSignatureChecksums.forEach(checksum => { }); }); - describe('validateChecksumHeaders', () => { passingCases.forEach(testCase => { it(testCase.description, () => { diff --git a/tests/unit/api/apiUtils/versioning.js b/tests/unit/api/apiUtils/versioning.js index 1edea6d0b0..1bd1e876f0 100644 --- a/tests/unit/api/apiUtils/versioning.js +++ b/tests/unit/api/apiUtils/versioning.js @@ -6,11 +6,13 @@ const INF_VID = versioning.VersionID.getInfVid(config.replicationGroupId); const { scaledMsPerDay } = config.getTimeOptions(); const sinon = require('sinon'); -const { processVersioningState, getMasterState, - getVersionSpecificMetadataOptions, - preprocessingVersioningDelete, - overwritingVersioning } = - require('../../../../lib/api/apiUtils/object/versioning'); +const { + processVersioningState, + getMasterState, + getVersionSpecificMetadataOptions, + preprocessingVersioningDelete, + overwritingVersioning, +} = require('../../../../lib/api/apiUtils/object/versioning'); describe('versioning helpers', () => { describe('getMasterState+processVersioningState', () => { @@ -518,17 +520,22 @@ describe('versioning helpers', () => { }, ].forEach(testCase => [false, true].forEach(nullVersionCompatMode => - ['Enabled', 'Suspended'].forEach(versioningStatus => it( - `${testCase.description}${nullVersionCompatMode ? ' (null compat)' : ''}` + - `, versioning Status=${versioningStatus}`, - () => { - const mst = getMasterState(testCase.objMD); - const res = processVersioningState(mst, versioningStatus, nullVersionCompatMode); - const resultName = `versioning${versioningStatus}` + - `${nullVersionCompatMode ? 'Compat' : ''}ExpectedRes`; - const expectedRes = testCase[resultName]; - assert.deepStrictEqual(res, expectedRes); - })))); + ['Enabled', 'Suspended'].forEach(versioningStatus => + it( + `${testCase.description}${nullVersionCompatMode ? ' (null compat)' : ''}` + + `, versioning Status=${versioningStatus}`, + () => { + const mst = getMasterState(testCase.objMD); + const res = processVersioningState(mst, versioningStatus, nullVersionCompatMode); + const resultName = + `versioning${versioningStatus}` + `${nullVersionCompatMode ? 'Compat' : ''}ExpectedRes`; + const expectedRes = testCase[resultName]; + assert.deepStrictEqual(res, expectedRes); + }, + ), + ), + ), + ); }); describe('getVersionSpecificMetadataOptions', () => { @@ -583,14 +590,13 @@ describe('versioning helpers', () => { }, ].forEach(testCase => [false, true].forEach(nullVersionCompatMode => - it(`${testCase.description}${nullVersionCompatMode ? ' (null compat)' : ''}`, - () => { - const options = getVersionSpecificMetadataOptions( - testCase.objMD, nullVersionCompatMode); - const expectedResAttr = nullVersionCompatMode ? - 'expectedResCompat' : 'expectedRes'; + it(`${testCase.description}${nullVersionCompatMode ? ' (null compat)' : ''}`, () => { + const options = getVersionSpecificMetadataOptions(testCase.objMD, nullVersionCompatMode); + const expectedResAttr = nullVersionCompatMode ? 'expectedResCompat' : 'expectedRes'; assert.deepStrictEqual(options, testCase[expectedResAttr]); - }))); + }), + ), + ); }); describe('preprocessingVersioningDelete', () => { @@ -669,24 +675,28 @@ describe('versioning helpers', () => { }, ].forEach(testCase => [false, true].forEach(nullVersionCompatMode => - it(`${testCase.description}${nullVersionCompatMode ? ' (null compat)' : ''}`, - () => { + it(`${testCase.description}${nullVersionCompatMode ? ' (null compat)' : ''}`, () => { const mockBucketMD = { getVersioningConfiguration: () => ({ Status: 'Enabled' }), }; const options = preprocessingVersioningDelete( - 'foobucket', mockBucketMD, testCase.objMD, testCase.reqVersionId, - nullVersionCompatMode); - const expectedResAttr = nullVersionCompatMode ? - 'expectedResCompat' : 'expectedRes'; + 'foobucket', + mockBucketMD, + testCase.objMD, + testCase.reqVersionId, + nullVersionCompatMode, + ); + const expectedResAttr = nullVersionCompatMode ? 'expectedResCompat' : 'expectedRes'; assert.deepStrictEqual(options, testCase[expectedResAttr]); - }))); + }), + ), + ); }); describe('overwritingVersioning', () => { const days = 3; const archiveInfo = { - 'archiveID': '126783123678', + archiveID: '126783123678', }; const now = Date.now(); let clock; @@ -702,70 +712,70 @@ describe('versioning helpers', () => { [ { description: 'Should update archive with restore infos', - objMD: { - 'versionId': '2345678', + objMD: { + versionId: '2345678', 'creation-time': now, 'last-modified': now, - 'originOp': 's3:PutObject', + originOp: 's3:PutObject', 'x-amz-storage-class': 'cold-location', - 'archive': { - 'restoreRequestedDays': days, - 'restoreRequestedAt': now, - archiveInfo - } + archive: { + restoreRequestedDays: days, + restoreRequestedAt: now, + archiveInfo, + }, }, expectedRes: { - 'creationTime': now, - 'lastModifiedDate': now, - 'updateMicroVersionId': true, - 'originOp': 's3:ObjectRestore:Completed', - 'taggingCopy': undefined, - 'amzStorageClass': 'cold-location', - 'archive': { + creationTime: now, + lastModifiedDate: now, + updateMicroVersionId: true, + originOp: 's3:ObjectRestore:Completed', + taggingCopy: undefined, + amzStorageClass: 'cold-location', + archive: { archiveInfo, - 'restoreRequestedDays': 3, - 'restoreRequestedAt': now, - 'restoreCompletedAt': new Date(now), - 'restoreWillExpireAt': new Date(now + (days * scaledMsPerDay)), - } - } + restoreRequestedDays: 3, + restoreRequestedAt: now, + restoreCompletedAt: new Date(now), + restoreWillExpireAt: new Date(now + days * scaledMsPerDay), + }, + }, }, { description: 'Should keep user mds and tags', hasUserMD: true, objMD: { - 'versionId': '2345678', + versionId: '2345678', 'creation-time': now, 'last-modified': now, - 'originOp': 's3:PutObject', + originOp: 's3:PutObject', 'x-amz-meta-test': 'test', 'x-amz-meta-test2': 'test2', - 'tags': { 'testtag': 'testtag', 'testtag2': 'testtag2' }, + tags: { testtag: 'testtag', testtag2: 'testtag2' }, 'x-amz-storage-class': 'cold-location', - 'archive': { - 'restoreRequestedDays': days, - 'restoreRequestedAt': now, - archiveInfo - } + archive: { + restoreRequestedDays: days, + restoreRequestedAt: now, + archiveInfo, + }, }, expectedRes: { - 'creationTime': now, - 'lastModifiedDate': now, - 'updateMicroVersionId': true, - 'originOp': 's3:ObjectRestore:Completed', - 'metaHeaders': { + creationTime: now, + lastModifiedDate: now, + updateMicroVersionId: true, + originOp: 's3:ObjectRestore:Completed', + metaHeaders: { 'x-amz-meta-test': 'test', 'x-amz-meta-test2': 'test2', }, - 'taggingCopy': { 'testtag': 'testtag', 'testtag2': 'testtag2' }, - 'amzStorageClass': 'cold-location', - 'archive': { + taggingCopy: { testtag: 'testtag', testtag2: 'testtag2' }, + amzStorageClass: 'cold-location', + archive: { archiveInfo, - 'restoreRequestedDays': days, - 'restoreRequestedAt': now, - 'restoreCompletedAt': new Date(now), - 'restoreWillExpireAt': new Date(now + (days * scaledMsPerDay)), - } + restoreRequestedDays: days, + restoreRequestedAt: now, + restoreCompletedAt: new Date(now), + restoreWillExpireAt: new Date(now + days * scaledMsPerDay), + }, }, }, { @@ -773,257 +783,243 @@ describe('versioning helpers', () => { objMD: { 'creation-time': now, 'last-modified': now, - 'originOp': 's3:PutObject', - 'nullVersionId': 'vnull', - 'isNull': true, + originOp: 's3:PutObject', + nullVersionId: 'vnull', + isNull: true, 'x-amz-storage-class': 'cold-location', - 'archive': { - 'restoreRequestedDays': days, - 'restoreRequestedAt': now, - archiveInfo - } + archive: { + restoreRequestedDays: days, + restoreRequestedAt: now, + archiveInfo, + }, }, expectedRes: { - 'creationTime': now, - 'lastModifiedDate': now, - 'updateMicroVersionId': true, - 'originOp': 's3:ObjectRestore:Completed', - 'amzStorageClass': 'cold-location', - 'taggingCopy': undefined, - 'archive': { + creationTime: now, + lastModifiedDate: now, + updateMicroVersionId: true, + originOp: 's3:ObjectRestore:Completed', + amzStorageClass: 'cold-location', + taggingCopy: undefined, + archive: { archiveInfo, - 'restoreRequestedDays': 3, - 'restoreRequestedAt': now, - 'restoreCompletedAt': new Date(now), - 'restoreWillExpireAt': new Date(now + (days * scaledMsPerDay)), - } - } + restoreRequestedDays: 3, + restoreRequestedAt: now, + restoreCompletedAt: new Date(now), + restoreWillExpireAt: new Date(now + days * scaledMsPerDay), + }, + }, }, { description: 'Should not keep x-amz-meta-scal-s3-restore-attempt user MD', hasUserMD: true, objMD: { - 'versionId': '2345678', + versionId: '2345678', 'creation-time': now, 'last-modified': now, - 'originOp': 's3:PutObject', + originOp: 's3:PutObject', 'x-amz-meta-test': 'test', 'x-amz-meta-scal-s3-restore-attempt': 14, 'x-amz-storage-class': 'cold-location', - 'archive': { - 'restoreRequestedDays': days, - 'restoreRequestedAt': now, - archiveInfo - } + archive: { + restoreRequestedDays: days, + restoreRequestedAt: now, + archiveInfo, + }, }, expectedRes: { - 'creationTime': now, - 'lastModifiedDate': now, - 'updateMicroVersionId': true, - 'originOp': 's3:ObjectRestore:Completed', - 'metaHeaders': { + creationTime: now, + lastModifiedDate: now, + updateMicroVersionId: true, + originOp: 's3:ObjectRestore:Completed', + metaHeaders: { 'x-amz-meta-test': 'test', }, - 'taggingCopy': undefined, - 'amzStorageClass': 'cold-location', - 'archive': { + taggingCopy: undefined, + amzStorageClass: 'cold-location', + archive: { archiveInfo, - 'restoreRequestedDays': 3, - 'restoreRequestedAt': now, - 'restoreCompletedAt': new Date(now), - 'restoreWillExpireAt': new Date(now + (days * scaledMsPerDay)), - } - } + restoreRequestedDays: 3, + restoreRequestedAt: now, + restoreCompletedAt: new Date(now), + restoreWillExpireAt: new Date(now + days * scaledMsPerDay), + }, + }, }, { description: 'Should keep replication infos', objMD: { - 'versionId': '2345678', - 'creation-time': now, - 'last-modified': now, - 'originOp': 's3:PutObject', - 'x-amz-storage-class': 'cold-location', - 'replicationInfo': { - 'status': 'COMPLETED', - 'backends': [ - { - 'site': 'azure-blob', - 'status': 'COMPLETED', - 'dataStoreVersionId': '' - } - ], - 'content': [ - 'DATA', - 'METADATA' - ], - 'destination': 'arn:aws:s3:::replicate-cold', - 'storageClass': 'azure-blob', - 'role': 'arn:aws:iam::root:role/s3-replication-role', - 'storageType': 'azure', - 'dataStoreVersionId': '', - }, - archive: { - 'restoreRequestedDays': days, - 'restoreRequestedAt': now, - archiveInfo - } + versionId: '2345678', + 'creation-time': now, + 'last-modified': now, + originOp: 's3:PutObject', + 'x-amz-storage-class': 'cold-location', + replicationInfo: { + status: 'COMPLETED', + backends: [ + { + site: 'azure-blob', + status: 'COMPLETED', + dataStoreVersionId: '', + }, + ], + content: ['DATA', 'METADATA'], + destination: 'arn:aws:s3:::replicate-cold', + storageClass: 'azure-blob', + role: 'arn:aws:iam::root:role/s3-replication-role', + storageType: 'azure', + dataStoreVersionId: '', + }, + archive: { + restoreRequestedDays: days, + restoreRequestedAt: now, + archiveInfo, + }, }, expectedRes: { - 'creationTime': now, - 'lastModifiedDate': now, - 'updateMicroVersionId': true, - 'originOp': 's3:ObjectRestore:Completed', - 'amzStorageClass': 'cold-location', - 'replicationInfo': { - 'status': 'COMPLETED', - 'backends': [ + creationTime: now, + lastModifiedDate: now, + updateMicroVersionId: true, + originOp: 's3:ObjectRestore:Completed', + amzStorageClass: 'cold-location', + replicationInfo: { + status: 'COMPLETED', + backends: [ { - 'site': 'azure-blob', - 'status': 'COMPLETED', - 'dataStoreVersionId': '' - } + site: 'azure-blob', + status: 'COMPLETED', + dataStoreVersionId: '', + }, ], - 'content': [ - 'DATA', - 'METADATA' - ], - 'destination': 'arn:aws:s3:::replicate-cold', - 'storageClass': 'azure-blob', - 'role': 'arn:aws:iam::root:role/s3-replication-role', - 'storageType': 'azure', - 'dataStoreVersionId': '', - }, - 'taggingCopy': undefined, + content: ['DATA', 'METADATA'], + destination: 'arn:aws:s3:::replicate-cold', + storageClass: 'azure-blob', + role: 'arn:aws:iam::root:role/s3-replication-role', + storageType: 'azure', + dataStoreVersionId: '', + }, + taggingCopy: undefined, archive: { archiveInfo, - 'restoreRequestedDays': 3, - 'restoreRequestedAt': now, - 'restoreCompletedAt': new Date(now), - 'restoreWillExpireAt': new Date(now + (days * scaledMsPerDay)), - } - } + restoreRequestedDays: 3, + restoreRequestedAt: now, + restoreCompletedAt: new Date(now), + restoreWillExpireAt: new Date(now + days * scaledMsPerDay), + }, + }, }, { description: 'Should keep legalHold', objMD: { - 'versionId': '2345678', - 'creation-time': now, - 'last-modified': now, - 'originOp': 's3:PutObject', - 'legalHold': true, - 'x-amz-storage-class': 'cold-location', - 'archive': { - 'restoreRequestedDays': days, - 'restoreRequestedAt': now, - archiveInfo - } + versionId: '2345678', + 'creation-time': now, + 'last-modified': now, + originOp: 's3:PutObject', + legalHold: true, + 'x-amz-storage-class': 'cold-location', + archive: { + restoreRequestedDays: days, + restoreRequestedAt: now, + archiveInfo, + }, }, expectedRes: { - 'creationTime': now, - 'lastModifiedDate': now, - 'updateMicroVersionId': true, - 'originOp': 's3:ObjectRestore:Completed', - 'legalHold': true, - 'amzStorageClass': 'cold-location', - 'taggingCopy': undefined, - 'archive': { + creationTime: now, + lastModifiedDate: now, + updateMicroVersionId: true, + originOp: 's3:ObjectRestore:Completed', + legalHold: true, + amzStorageClass: 'cold-location', + taggingCopy: undefined, + archive: { archiveInfo, - 'restoreRequestedDays': 3, - 'restoreRequestedAt': now, - 'restoreCompletedAt': new Date(now), - 'restoreWillExpireAt': new Date(now + (days * scaledMsPerDay)), - } - } + restoreRequestedDays: 3, + restoreRequestedAt: now, + restoreCompletedAt: new Date(now), + restoreWillExpireAt: new Date(now + days * scaledMsPerDay), + }, + }, }, { description: 'Should keep ACLs', objMD: { - 'versionId': '2345678', - 'creation-time': now, - 'last-modified': now, - 'originOp': 's3:PutObject', - 'x-amz-storage-class': 'cold-location', - 'acl': { - 'Canned': '', - 'FULL_CONTROL': [ - '872c04772893deae2b48365752362cd92672eb80eb3deea50d89e834a10ce185' - ], - 'WRITE_ACP': [], - 'READ': [ - 'http://acs.amazonaws.com/groups/global/AllUsers' - ], - 'READ_ACP': [] - }, - 'archive': { - 'restoreRequestedDays': days, - 'restoreRequestedAt': now, - archiveInfo - } + versionId: '2345678', + 'creation-time': now, + 'last-modified': now, + originOp: 's3:PutObject', + 'x-amz-storage-class': 'cold-location', + acl: { + Canned: '', + FULL_CONTROL: ['872c04772893deae2b48365752362cd92672eb80eb3deea50d89e834a10ce185'], + WRITE_ACP: [], + READ: ['http://acs.amazonaws.com/groups/global/AllUsers'], + READ_ACP: [], + }, + archive: { + restoreRequestedDays: days, + restoreRequestedAt: now, + archiveInfo, + }, }, expectedRes: { - 'creationTime': now, - 'lastModifiedDate': now, - 'updateMicroVersionId': true, - 'originOp': 's3:ObjectRestore:Completed', - 'acl': { - 'Canned': '', - 'FULL_CONTROL': [ - '872c04772893deae2b48365752362cd92672eb80eb3deea50d89e834a10ce185' - ], - 'WRITE_ACP': [], - 'READ': [ - 'http://acs.amazonaws.com/groups/global/AllUsers' - ], - 'READ_ACP': [] - }, - 'taggingCopy': undefined, - 'amzStorageClass': 'cold-location', - 'archive': { + creationTime: now, + lastModifiedDate: now, + updateMicroVersionId: true, + originOp: 's3:ObjectRestore:Completed', + acl: { + Canned: '', + FULL_CONTROL: ['872c04772893deae2b48365752362cd92672eb80eb3deea50d89e834a10ce185'], + WRITE_ACP: [], + READ: ['http://acs.amazonaws.com/groups/global/AllUsers'], + READ_ACP: [], + }, + taggingCopy: undefined, + amzStorageClass: 'cold-location', + archive: { archiveInfo, - 'restoreRequestedDays': 3, - 'restoreRequestedAt': now, - 'restoreCompletedAt': new Date(now), - 'restoreWillExpireAt': new Date(now + (days * scaledMsPerDay)), - } + restoreRequestedDays: 3, + restoreRequestedAt: now, + restoreCompletedAt: new Date(now), + restoreWillExpireAt: new Date(now + days * scaledMsPerDay), + }, }, }, - { - description: 'Should keep contentMD5 of the original object', - objMD: { - 'versionId': '2345678', + { + description: 'Should keep contentMD5 of the original object', + objMD: { + versionId: '2345678', 'creation-time': now, 'last-modified': now, - 'originOp': 's3:PutObject', + originOp: 's3:PutObject', 'x-amz-storage-class': 'cold-location', 'content-md5': '123456789-5', - 'acl': {}, - 'archive': { - 'restoreRequestedDays': days, - 'restoreRequestedAt': now, - archiveInfo - } - }, - metadataStoreParams: { - 'contentMD5': '987654321-3', - }, - expectedRes: { - 'creationTime': now, - 'lastModifiedDate': now, - 'updateMicroVersionId': true, - 'originOp': 's3:ObjectRestore:Completed', - 'contentMD5': '123456789-5', - 'restoredEtag': '987654321-3', - 'acl': {}, - 'taggingCopy': undefined, - 'amzStorageClass': 'cold-location', - 'archive': { - archiveInfo, - 'restoreRequestedDays': 3, - 'restoreRequestedAt': now, - 'restoreCompletedAt': new Date(now), - 'restoreWillExpireAt': new Date(now + (days * scaledMsPerDay)), - } - } + acl: {}, + archive: { + restoreRequestedDays: days, + restoreRequestedAt: now, + archiveInfo, + }, + }, + metadataStoreParams: { + contentMD5: '987654321-3', + }, + expectedRes: { + creationTime: now, + lastModifiedDate: now, + updateMicroVersionId: true, + originOp: 's3:ObjectRestore:Completed', + contentMD5: '123456789-5', + restoredEtag: '987654321-3', + acl: {}, + taggingCopy: undefined, + amzStorageClass: 'cold-location', + archive: { + archiveInfo, + restoreRequestedDays: 3, + restoreRequestedAt: now, + restoreCompletedAt: new Date(now), + restoreWillExpireAt: new Date(now + days * scaledMsPerDay), + }, + }, }, ].forEach(testCase => { it(testCase.description, () => { diff --git a/tests/unit/api/bucketACLauth.js b/tests/unit/api/bucketACLauth.js index 125fed71d5..0f104ba948 100644 --- a/tests/unit/api/bucketACLauth.js +++ b/tests/unit/api/bucketACLauth.js @@ -1,8 +1,7 @@ const assert = require('assert'); const BucketInfo = require('arsenal').models.BucketInfo; const constants = require('../../../constants'); -const { isBucketAuthorized } - = require('../../../lib/api/apiUtils/authorization/permissionChecks'); +const { isBucketAuthorized } = require('../../../lib/api/apiUtils/authorization/permissionChecks'); const { DummyRequestLogger, makeAuthInfo } = require('../helpers'); const lifecycleServiceAccountId = '0123456789abcdef/lifecycle'; @@ -16,12 +15,10 @@ const ownerCanonicalId = authInfo.getCanonicalID(); const altAcctAuthInfo = makeAuthInfo(altAccessKey); const accountToVet = altAcctAuthInfo.getCanonicalID(); -const bucket = new BucketInfo('niftyBucket', ownerCanonicalId, - authInfo.getAccountDisplayName(), creationDate); +const bucket = new BucketInfo('niftyBucket', ownerCanonicalId, authInfo.getAccountDisplayName(), creationDate); const log = new DummyRequestLogger(); -describe('bucket authorization for bucketGet, bucketHead, ' + - 'objectGet, and objectHead', () => { +describe('bucket authorization for bucketGet, bucketHead, ' + 'objectGet, and objectHead', () => { // Reset the bucket ACLs afterEach(() => { bucket.setFullAcl({ @@ -44,70 +41,97 @@ describe('bucket authorization for bucketGet, bucketHead, ' + const orders = [ { - it: 'should allow access to bucket owner', canned: '', - id: ownerCanonicalId, response: trueArray, auth: authInfo, + it: 'should allow access to bucket owner', + canned: '', + id: ownerCanonicalId, + response: trueArray, + auth: authInfo, }, { - it: 'should allow access to user in bucket owner account', canned: '', - id: ownerCanonicalId, response: trueArray, auth: userAuthInfo, + it: 'should allow access to user in bucket owner account', + canned: '', + id: ownerCanonicalId, + response: trueArray, + auth: userAuthInfo, }, { it: 'should allow access to lifecycle service account', - canned: '', id: lifecycleServiceAccountId, response: trueArray, + canned: '', + id: lifecycleServiceAccountId, + response: trueArray, }, { - it: 'should allow public-user access for unknown ' + - 'service account and private canned ACL', - canned: '', id: unknownServiceAccountId, + it: 'should allow public-user access for unknown ' + 'service account and private canned ACL', + canned: '', + id: unknownServiceAccountId, response: falseArrayBucketTrueArrayObject, }, { it: 'should allow access to anyone if canned public-read ACL', - canned: 'public-read', id: accountToVet, response: trueArray, + canned: 'public-read', + id: accountToVet, + response: trueArray, auth: altAcctAuthInfo, }, { it: 'should allow access to anyone if canned public-read-write ACL', - canned: 'public-read-write', id: accountToVet, response: trueArray, + canned: 'public-read-write', + id: accountToVet, + response: trueArray, auth: altAcctAuthInfo, }, { - it: 'should not allow request on the bucket (bucketGet, bucketHead)' - + ' but should allow request on the object (objectGet, objectHead)' - + ' to public user if authenticated-read ACL', - canned: 'authenticated-read', id: constants.publicId, - response: falseArrayBucketTrueArrayObject, auth: altAcctAuthInfo, + it: + 'should not allow request on the bucket (bucketGet, bucketHead)' + + ' but should allow request on the object (objectGet, objectHead)' + + ' to public user if authenticated-read ACL', + canned: 'authenticated-read', + id: constants.publicId, + response: falseArrayBucketTrueArrayObject, + auth: altAcctAuthInfo, }, { - it: 'should allow access to any authenticated user if authenticated' - + '-read ACL', canned: 'authenticated-read', id: accountToVet, - response: trueArray, auth: altAcctAuthInfo, + it: 'should allow access to any authenticated user if authenticated' + '-read ACL', + canned: 'authenticated-read', + id: accountToVet, + response: trueArray, + auth: altAcctAuthInfo, }, { - it: 'should not allow request on the bucket (bucketGet, bucketHead)' - + ' but should allow request on the object (objectGet, objectHead)' - + ' to public user if private canned ACL', - canned: '', id: accountToVet, - response: falseArrayBucketTrueArrayObject, auth: altAcctAuthInfo, + it: + 'should not allow request on the bucket (bucketGet, bucketHead)' + + ' but should allow request on the object (objectGet, objectHead)' + + ' to public user if private canned ACL', + canned: '', + id: accountToVet, + response: falseArrayBucketTrueArrayObject, + auth: altAcctAuthInfo, }, { - it: 'should not allow request on the bucket (bucketGet, bucketHead)' - + ' but should allow request on the object (objectGet, objectHead)' - + ' to just any user if private canned ACL', - canned: '', id: accountToVet, - response: falseArrayBucketTrueArrayObject, auth: altAcctAuthInfo, + it: + 'should not allow request on the bucket (bucketGet, bucketHead)' + + ' but should allow request on the object (objectGet, objectHead)' + + ' to just any user if private canned ACL', + canned: '', + id: accountToVet, + response: falseArrayBucketTrueArrayObject, + auth: altAcctAuthInfo, }, { - it: 'should allow access to user if account was granted' - + ' FULL_CONTROL', - canned: '', id: accountToVet, response: trueArray, - aclParam: ['FULL_CONTROL', accountToVet], auth: altAcctAuthInfo, + it: 'should allow access to user if account was granted' + ' FULL_CONTROL', + canned: '', + id: accountToVet, + response: trueArray, + aclParam: ['FULL_CONTROL', accountToVet], + auth: altAcctAuthInfo, }, { - it: 'should not allow access to just any user if private' - + ' canned ACL', - canned: '', id: accountToVet, response: trueArray, - aclParam: ['READ', accountToVet], auth: altAcctAuthInfo, + it: 'should not allow access to just any user if private' + ' canned ACL', + canned: '', + id: accountToVet, + response: trueArray, + aclParam: ['READ', accountToVet], + auth: altAcctAuthInfo, }, ]; @@ -117,8 +141,7 @@ describe('bucket authorization for bucketGet, bucketHead, ' + bucket.setSpecificAcl(value.aclParam[1], value.aclParam[0]); } bucket.setCannedAcl(value.canned); - const results = requestTypes.map(type => - isBucketAuthorized(bucket, type, value.id, value.auth, log)); + const results = requestTypes.map(type => isBucketAuthorized(bucket, type, value.id, value.auth, log)); assert.deepStrictEqual(results, value.response); done(); }); @@ -139,45 +162,45 @@ describe('bucket authorization for bucketGetACL', () => { }); it('should allow access to bucket owner', () => { - const result = isBucketAuthorized(bucket, 'bucketGetACL', - ownerCanonicalId, authInfo); + const result = isBucketAuthorized(bucket, 'bucketGetACL', ownerCanonicalId, authInfo); assert.strictEqual(result, true); }); it('should allow access to user in bucket owner account', () => { - const result = isBucketAuthorized(bucket, 'bucketGetACL', - ownerCanonicalId, userAuthInfo); + const result = isBucketAuthorized(bucket, 'bucketGetACL', ownerCanonicalId, userAuthInfo); assert.strictEqual(result, true); }); const orders = [ { it: 'log group only if canned log-delivery-write acl', - id: constants.logId, canned: 'log-delivery-write', auth: null, + id: constants.logId, + canned: 'log-delivery-write', + auth: null, }, { it: 'account only if account was granted FULL_CONTROL right', - id: accountToVet, aclParam: ['FULL_CONTROL', accountToVet], + id: accountToVet, + aclParam: ['FULL_CONTROL', accountToVet], auth: altAcctAuthInfo, }, { it: 'account only if account was granted READ_ACP right', - id: accountToVet, aclParam: ['READ_ACP', accountToVet], + id: accountToVet, + aclParam: ['READ_ACP', accountToVet], auth: altAcctAuthInfo, }, ]; orders.forEach(value => { it(`should allow access to ${value.it}`, done => { - const noAuthResult = isBucketAuthorized(bucket, 'bucketGetACL', - value.id); + const noAuthResult = isBucketAuthorized(bucket, 'bucketGetACL', value.id); assert.strictEqual(noAuthResult, false); if (value.aclParam) { bucket.setSpecificAcl(value.aclParam[1], value.aclParam[0]); } else if (value.canned) { bucket.setCannedAcl(value.canned); } - const authorizedResult = isBucketAuthorized(bucket, 'bucketGetACL', - value.id, value.auth); + const authorizedResult = isBucketAuthorized(bucket, 'bucketGetACL', value.id, value.auth); assert.strictEqual(authorizedResult, true); done(); }); @@ -198,27 +221,22 @@ describe('bucket authorization for bucketPutACL', () => { }); it('should allow access to bucket owner', () => { - const result = isBucketAuthorized(bucket, 'bucketPutACL', - ownerCanonicalId, authInfo); + const result = isBucketAuthorized(bucket, 'bucketPutACL', ownerCanonicalId, authInfo); assert.strictEqual(result, true); }); it('should allow access to user in bucket owner account', () => { - const result = isBucketAuthorized(bucket, 'bucketPutACL', - ownerCanonicalId, userAuthInfo); + const result = isBucketAuthorized(bucket, 'bucketPutACL', ownerCanonicalId, userAuthInfo); assert.strictEqual(result, true); }); const orders = ['FULL_CONTROL', 'WRITE_ACP']; orders.forEach(value => { - it('should allow access to account if ' + - `account was granted ${value} right`, done => { - const noAuthResult = isBucketAuthorized(bucket, 'bucketPutACL', - accountToVet, altAcctAuthInfo); + it('should allow access to account if ' + `account was granted ${value} right`, done => { + const noAuthResult = isBucketAuthorized(bucket, 'bucketPutACL', accountToVet, altAcctAuthInfo); assert.strictEqual(noAuthResult, false); bucket.setSpecificAcl(accountToVet, value); - const authorizedResult = isBucketAuthorized(bucket, 'bucketPutACL', - accountToVet, altAcctAuthInfo); + const authorizedResult = isBucketAuthorized(bucket, 'bucketPutACL', accountToVet, altAcctAuthInfo); assert.strictEqual(authorizedResult, true); done(); }); @@ -239,26 +257,28 @@ describe('bucket authorization for bucketOwnerAction', () => { }); it('should allow access to bucket owner', () => { - const result = isBucketAuthorized(bucket, 'bucketDeleteCors', - ownerCanonicalId, authInfo); + const result = isBucketAuthorized(bucket, 'bucketDeleteCors', ownerCanonicalId, authInfo); assert.strictEqual(result, true); }); it('should allow access to user in bucket owner account', () => { - const result = isBucketAuthorized(bucket, 'bucketDeleteCors', - ownerCanonicalId, userAuthInfo); + const result = isBucketAuthorized(bucket, 'bucketDeleteCors', ownerCanonicalId, userAuthInfo); assert.strictEqual(result, true); }); const orders = [ { - it: 'other account (even if other account has FULL_CONTROL rights' - + ' in bucket)', id: accountToVet, canned: '', - aclParam: ['FULL_CONTROL', accountToVet], auth: altAcctAuthInfo, + it: 'other account (even if other account has FULL_CONTROL rights' + ' in bucket)', + id: accountToVet, + canned: '', + aclParam: ['FULL_CONTROL', accountToVet], + auth: altAcctAuthInfo, }, { it: 'public user (even if bucket is public read write)', - id: constants.publicId, canned: 'public-read-write', auth: altAcctAuthInfo, + id: constants.publicId, + canned: 'public-read-write', + auth: altAcctAuthInfo, }, ]; orders.forEach(value => { @@ -267,8 +287,7 @@ describe('bucket authorization for bucketOwnerAction', () => { bucket.setSpecificAcl(value.aclParam[1], value.aclParam[0]); } bucket.setCannedAcl(value.canned); - const result = isBucketAuthorized(bucket, 'bucketDeleteCors', - value.id, value.auth); + const result = isBucketAuthorized(bucket, 'bucketDeleteCors', value.id, value.auth); assert.strictEqual(result, false); done(); }); @@ -289,26 +308,28 @@ describe('bucket authorization for bucketDelete', () => { }); it('should allow access to bucket owner', () => { - const result = isBucketAuthorized(bucket, 'bucketDelete', - ownerCanonicalId, authInfo); + const result = isBucketAuthorized(bucket, 'bucketDelete', ownerCanonicalId, authInfo); assert.strictEqual(result, true); }); it('should allow access to user in bucket owner account', () => { - const result = isBucketAuthorized(bucket, 'bucketDelete', - ownerCanonicalId, userAuthInfo); + const result = isBucketAuthorized(bucket, 'bucketDelete', ownerCanonicalId, userAuthInfo); assert.strictEqual(result, true); }); const orders = [ { - it: 'other account (even if other account has FULL_CONTROL rights ' - + 'in bucket)', id: accountToVet, canned: '', - aclParam: ['FULL_CONTROL', accountToVet], auth: altAcctAuthInfo, + it: 'other account (even if other account has FULL_CONTROL rights ' + 'in bucket)', + id: accountToVet, + canned: '', + aclParam: ['FULL_CONTROL', accountToVet], + auth: altAcctAuthInfo, }, { it: 'public user (even if bucket is public read write)', - id: constants.publicId, canned: 'public-read-write', auth: null, + id: constants.publicId, + canned: 'public-read-write', + auth: null, }, ]; orders.forEach(value => { @@ -340,14 +361,12 @@ describe('bucket authorization for objectDelete and objectPut', () => { const requestTypes = ['objectDelete', 'objectPut']; it('should allow access to bucket owner', () => { - const results = requestTypes.map(type => - isBucketAuthorized(bucket, type, ownerCanonicalId, authInfo)); + const results = requestTypes.map(type => isBucketAuthorized(bucket, type, ownerCanonicalId, authInfo)); assert.deepStrictEqual(results, [true, true]); }); it('should allow access to user in bucket owner account', () => { - const results = requestTypes.map(type => - isBucketAuthorized(bucket, type, ownerCanonicalId, userAuthInfo)); + const results = requestTypes.map(type => isBucketAuthorized(bucket, type, ownerCanonicalId, userAuthInfo)); assert.deepStrictEqual(results, [true, true]); }); @@ -355,45 +374,50 @@ describe('bucket authorization for objectDelete and objectPut', () => { // NOTE objectPut is not needed for lifecycle but still // allowed, we would want more fine-grained implementation of // ACLs for service accounts later. - const results = requestTypes.map(type => - isBucketAuthorized(bucket, type, lifecycleServiceAccountId)); + const results = requestTypes.map(type => isBucketAuthorized(bucket, type, lifecycleServiceAccountId)); assert.deepStrictEqual(results, [true, true]); }); it('should deny access to unknown service account', () => { - const results = requestTypes.map(type => - isBucketAuthorized(bucket, type, unknownServiceAccountId)); + const results = requestTypes.map(type => isBucketAuthorized(bucket, type, unknownServiceAccountId)); assert.deepStrictEqual(results, [false, false]); }); const orders = [ { it: 'anyone if canned public-read-write ACL', - canned: 'public-read-write', id: constants.publicId, + canned: 'public-read-write', + id: constants.publicId, response: [true, true], }, { - it: 'user if account was granted FULL_CONTROL', canned: '', - id: accountToVet, response: [false, false], - aclParam: ['FULL_CONTROL', accountToVet], auth: altAcctAuthInfo, + it: 'user if account was granted FULL_CONTROL', + canned: '', + id: accountToVet, + response: [false, false], + aclParam: ['FULL_CONTROL', accountToVet], + auth: altAcctAuthInfo, }, { - it: 'user if account was granted WRITE right', canned: '', - id: accountToVet, response: [false, false], - aclParam: ['WRITE', accountToVet], auth: altAcctAuthInfo, + it: 'user if account was granted WRITE right', + canned: '', + id: accountToVet, + response: [false, false], + aclParam: ['WRITE', accountToVet], + auth: altAcctAuthInfo, }, ]; orders.forEach(value => { it(`should allow access to ${value.it}`, done => { bucket.setCannedAcl(value.canned); - const noAuthResults = requestTypes.map(type => - isBucketAuthorized(bucket, type, value.id, value.auth)); + const noAuthResults = requestTypes.map(type => isBucketAuthorized(bucket, type, value.id, value.auth)); assert.deepStrictEqual(noAuthResults, value.response); if (value.aclParam) { bucket.setSpecificAcl(value.aclParam[1], value.aclParam[0]); } const authResults = requestTypes.map(type => - isBucketAuthorized(bucket, type, accountToVet, altAcctAuthInfo)); + isBucketAuthorized(bucket, type, accountToVet, altAcctAuthInfo), + ); assert.deepStrictEqual(authResults, [true, true]); done(); }); @@ -401,14 +425,11 @@ describe('bucket authorization for objectDelete and objectPut', () => { }); describe('bucket authorization for objectPutACL and objectGetACL', () => { - it('should allow access to anyone since checks ' + - 'are done at object level', done => { + it('should allow access to anyone since checks ' + 'are done at object level', done => { const requestTypes = ['objectPutACL', 'objectGetACL']; - const results = requestTypes.map(type => - isBucketAuthorized(bucket, type, accountToVet, altAcctAuthInfo)); + const results = requestTypes.map(type => isBucketAuthorized(bucket, type, accountToVet, altAcctAuthInfo)); assert.deepStrictEqual(results, [true, true]); - const publicUserResults = requestTypes.map(type => - isBucketAuthorized(bucket, type, constants.publicId)); + const publicUserResults = requestTypes.map(type => isBucketAuthorized(bucket, type, constants.publicId)); assert.deepStrictEqual(publicUserResults, [true, true]); done(); }); diff --git a/tests/unit/api/bucketDelete.js b/tests/unit/api/bucketDelete.js index 6cd7d580ba..9442b8e64b 100644 --- a/tests/unit/api/bucketDelete.js +++ b/tests/unit/api/bucketDelete.js @@ -11,8 +11,7 @@ const { bucketPut } = require('../../../lib/api/bucketPut'); const bucketPutEncryption = require('../../../lib/api/bucketPutEncryption'); const { templateSSEConfig, templateRequest } = require('../utils/bucketEncryption'); const constants = require('../../../constants'); -const initiateMultipartUpload - = require('../../../lib/api/initiateMultipartUpload'); +const initiateMultipartUpload = require('../../../lib/api/initiateMultipartUpload'); const metadata = require('../metadataswitch'); const metadataMem = require('arsenal').storage.metadata.inMemory.metadata; const objectPut = require('../../../lib/api/objectPut'); @@ -20,7 +19,6 @@ const objectPutPart = require('../../../lib/api/objectPutPart'); const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../helpers'); const DummyRequest = require('../DummyRequest'); - const log = new DummyRequestLogger(); const canonicalID = 'accessKey1'; const authInfo = makeAuthInfo(canonicalID); @@ -32,55 +30,59 @@ const objectName = 'objectName'; const mpuBucket = `${constants.mpuBucketPrefix}${bucketName}`; function createMPU(testRequest, initiateRequest, deleteOverviewMPUObj, cb) { - async.waterfall([ - next => bucketPut(authInfo, testRequest, log, next), - (corsHeaders, next) => initiateMultipartUpload(authInfo, - initiateRequest, log, next), - (result, corsHeaders, next) => { - parseString(result, next); - }, - (json, next) => { - const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; - const md5Hash = crypto.createHash('md5'); - const bufferBody = Buffer.from(postBody); - md5Hash.update(bufferBody); - const calculatedHash = md5Hash.digest('hex'); - const partRequest = new DummyRequest({ - bucketName, - objectKey: objectName, - namespace, - url: `/${objectName}?partNumber=1&uploadId=${testUploadId}`, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - query: { - partNumber: '1', - uploadId: testUploadId, - }, - calculatedHash, - }, postBody); - objectPutPart(authInfo, partRequest, undefined, log, err => { - if (err) { - return next(err); - } - return next(null, testUploadId); + async.waterfall( + [ + next => bucketPut(authInfo, testRequest, log, next), + (corsHeaders, next) => initiateMultipartUpload(authInfo, initiateRequest, log, next), + (result, corsHeaders, next) => { + parseString(result, next); + }, + (json, next) => { + const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; + const md5Hash = crypto.createHash('md5'); + const bufferBody = Buffer.from(postBody); + md5Hash.update(bufferBody); + const calculatedHash = md5Hash.digest('hex'); + const partRequest = new DummyRequest( + { + bucketName, + objectKey: objectName, + namespace, + url: `/${objectName}?partNumber=1&uploadId=${testUploadId}`, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + query: { + partNumber: '1', + uploadId: testUploadId, + }, + calculatedHash, + }, + postBody, + ); + objectPutPart(authInfo, partRequest, undefined, log, err => { + if (err) { + return next(err); + } + return next(null, testUploadId); + }); + }, + ], + (err, testUploadId) => { + assert.strictEqual(err, null); + const mpuBucketKeyMap = metadataMem.metadata.keyMaps.get(mpuBucket); + assert.strictEqual(mpuBucketKeyMap.size, 2); + if (deleteOverviewMPUObj) { + const overviewKey = + `overview${constants.splitter}` + `${objectName}${constants.splitter}${testUploadId}`; + // remove overview key from in mem mpu bucket + mpuBucketKeyMap.delete(overviewKey); + assert.strictEqual(mpuBucketKeyMap.size, 1); + } + bucketDelete(authInfo, testRequest, log, err => { + assert.strictEqual(err, null); + cb(); }); }, - ], (err, testUploadId) => { - assert.strictEqual(err, null); - const mpuBucketKeyMap = - metadataMem.metadata.keyMaps.get(mpuBucket); - assert.strictEqual(mpuBucketKeyMap.size, 2); - if (deleteOverviewMPUObj) { - const overviewKey = `overview${constants.splitter}` + - `${objectName}${constants.splitter}${testUploadId}`; - // remove overview key from in mem mpu bucket - mpuBucketKeyMap.delete(overviewKey); - assert.strictEqual(mpuBucketKeyMap.size, 1); - } - bucketDelete(authInfo, testRequest, log, err => { - assert.strictEqual(err, null); - cb(); - }); - }); + ); } describe('bucketDelete API', () => { @@ -106,13 +108,16 @@ describe('bucketDelete API', () => { }; it('should return an error if the bucket is not empty', done => { - const testPutObjectRequest = new DummyRequest({ - bucketName, - headers: {}, - url: `/${bucketName}/${objectName}`, - namespace, - objectKey: objectName, - }, postBody); + const testPutObjectRequest = new DummyRequest( + { + bucketName, + headers: {}, + url: `/${bucketName}/${objectName}`, + namespace, + objectKey: objectName, + }, + postBody, + ); bucketPut(authInfo, testRequest, log, err => { assert.strictEqual(err, null); @@ -122,21 +127,22 @@ describe('bucketDelete API', () => { assert.strictEqual(err.is.BucketNotEmpty, true); metadata.getBucket(bucketName, log, (err, md) => { assert.strictEqual(md.getName(), bucketName); - metadata.listObject(usersBucket, + metadata.listObject( + usersBucket, { prefix: authInfo.getCanonicalID() }, - log, (err, listResponse) => { - assert.strictEqual(listResponse.Contents.length, - 1); + log, + (err, listResponse) => { + assert.strictEqual(listResponse.Contents.length, 1); done(); - }); + }, + ); }); }); }); }); }); - it('should not return an error if the bucket has an initiated mpu', - done => { + it('should not return an error if the bucket has an initiated mpu', done => { bucketPut(authInfo, testRequest, log, err => { assert.strictEqual(err, null); initiateMultipartUpload(authInfo, initiateRequest, log, err => { @@ -155,23 +161,21 @@ describe('bucketDelete API', () => { metadata.getBucket(bucketName, log, (err, md) => { assert.strictEqual(err.is.NoSuchBucket, true); assert.strictEqual(md, undefined); - metadata.listObject(usersBucket, { prefix: canonicalID }, - log, (err, listResponse) => { - assert.strictEqual(listResponse.Contents.length, 0); - done(); - }); + metadata.listObject(usersBucket, { prefix: canonicalID }, log, (err, listResponse) => { + assert.strictEqual(listResponse.Contents.length, 0); + done(); + }); }); }); }); }); - it('should delete a bucket even if the bucket has ongoing mpu', - done => createMPU(testRequest, initiateRequest, false, done)); + it('should delete a bucket even if the bucket has ongoing mpu', done => + createMPU(testRequest, initiateRequest, false, done)); // if only part object (and no overview objects) is in mpu shadow bucket - it('should delete a bucket even if the bucket has an orphan part', - done => createMPU(testRequest, initiateRequest, true, done)); - + it('should delete a bucket even if the bucket has an orphan part', done => + createMPU(testRequest, initiateRequest, true, done)); it('should prevent anonymous user delete bucket API access', done => { const publicAuthInfo = makeAuthInfo(constants.publicId); diff --git a/tests/unit/api/bucketDeleteCors.js b/tests/unit/api/bucketDeleteCors.js index 981fd62a8d..4a4f63aac2 100644 --- a/tests/unit/api/bucketDeleteCors.js +++ b/tests/unit/api/bucketDeleteCors.js @@ -3,10 +3,7 @@ const assert = require('assert'); const { bucketPut } = require('../../../lib/api/bucketPut'); const bucketPutCors = require('../../../lib/api/bucketPutCors'); const bucketDeleteCors = require('../../../lib/api/bucketDeleteCors'); -const { cleanup, - DummyRequestLogger, - makeAuthInfo, - CorsConfigTester } = require('../helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo, CorsConfigTester } = require('../helpers'); const metadata = require('../../../lib/metadata/wrapper'); const log = new DummyRequestLogger(); @@ -21,10 +18,8 @@ const testBucketPutRequest = { url: '/', actionImplicitDenies: false, }; -const testBucketPutCorsRequest = - corsUtil.createBucketCorsRequest('PUT', bucketName); -const testBucketDeleteCorsRequest = - corsUtil.createBucketCorsRequest('DELETE', bucketName); +const testBucketPutCorsRequest = corsUtil.createBucketCorsRequest('PUT', bucketName); +const testBucketDeleteCorsRequest = corsUtil.createBucketCorsRequest('DELETE', bucketName); describe('deleteBucketCors API', () => { beforeEach(done => { @@ -35,9 +30,8 @@ describe('deleteBucketCors API', () => { }); afterEach(() => cleanup()); - it('should delete a bucket\'s cors configuration in metadata', done => { - bucketDeleteCors(authInfo, testBucketDeleteCorsRequest, log, - err => { + it("should delete a bucket's cors configuration in metadata", done => { + bucketDeleteCors(authInfo, testBucketDeleteCorsRequest, log, err => { if (err) { process.stdout.write(`Unexpected err ${err}`); return done(err); diff --git a/tests/unit/api/bucketDeleteEncryption.js b/tests/unit/api/bucketDeleteEncryption.js index 13bf39bb6a..a8aa3c9f25 100644 --- a/tests/unit/api/bucketDeleteEncryption.js +++ b/tests/unit/api/bucketDeleteEncryption.js @@ -51,7 +51,8 @@ describe('bucketDeleteEncryption API', () => { }); }); }); - })); + }), + ); it('should remove sse and clear key for aws:kms with a configured master key id', done => { const post = templateSSEConfig({ algorithm: 'aws:kms', keyId: '12345' }); diff --git a/tests/unit/api/bucketDeleteLifecycle.js b/tests/unit/api/bucketDeleteLifecycle.js index 647ff09ba1..f7e5763a0e 100644 --- a/tests/unit/api/bucketDeleteLifecycle.js +++ b/tests/unit/api/bucketDeleteLifecycle.js @@ -4,10 +4,7 @@ const async = require('async'); const { bucketPut } = require('../../../lib/api/bucketPut'); const bucketPutLifecycle = require('../../../lib/api/bucketDeleteLifecycle'); const bucketDeleteLifecycle = require('../../../lib/api/bucketDeleteLifecycle'); -const { cleanup, - DummyRequestLogger, - makeAuthInfo } - = require('../helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../helpers'); const metadata = require('../../../lib/metadata/wrapper'); const log = new DummyRequestLogger(); @@ -22,12 +19,13 @@ function _makeRequest(includeXml) { actionImplicitDenies: false, }; if (includeXml) { - request.post = '' + - '' + - 'Enabled' + - '1' + - ''; + request.post = + '' + + '' + + 'Enabled' + + '1' + + ''; } return request; } @@ -44,17 +42,20 @@ describe('deleteBucketLifecycle API', () => { }); }); it('should delete bucket lifecycle', done => { - async.series([ - next => bucketPutLifecycle(authInfo, _makeRequest(true), log, next), - next => bucketDeleteLifecycle(authInfo, _makeRequest(), log, next), - // eslint-disable-next-line no-unused-vars - next => metadata.getBucket(bucketName, log, (err, bucket, raftSessionId) => next(err, bucket)), - ], (err, results) => { - assert.equal(err, null, `Expected success, got error: ${err}`); - const bucket = results[2]; - const lifecycleConfig = bucket.getLifecycleConfiguration(); - assert.equal(lifecycleConfig, null); - done(); - }); + async.series( + [ + next => bucketPutLifecycle(authInfo, _makeRequest(true), log, next), + next => bucketDeleteLifecycle(authInfo, _makeRequest(), log, next), + // eslint-disable-next-line no-unused-vars + next => metadata.getBucket(bucketName, log, (err, bucket, raftSessionId) => next(err, bucket)), + ], + (err, results) => { + assert.equal(err, null, `Expected success, got error: ${err}`); + const bucket = results[2]; + const lifecycleConfig = bucket.getLifecycleConfiguration(); + assert.equal(lifecycleConfig, null); + done(); + }, + ); }); }); diff --git a/tests/unit/api/bucketDeletePolicy.js b/tests/unit/api/bucketDeletePolicy.js index 69bf9234db..fc683bcecd 100644 --- a/tests/unit/api/bucketDeletePolicy.js +++ b/tests/unit/api/bucketDeletePolicy.js @@ -4,10 +4,7 @@ const async = require('async'); const { bucketPut } = require('../../../lib/api/bucketPut'); const bucketPutPolicy = require('../../../lib/api/bucketPutPolicy'); const bucketDeletePolicy = require('../../../lib/api/bucketDeletePolicy'); -const { cleanup, - DummyRequestLogger, - makeAuthInfo } - = require('../helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../helpers'); const metadata = require('../../../lib/metadata/wrapper'); const log = new DummyRequestLogger(); @@ -50,17 +47,20 @@ describe('deleteBucketPolicy API', () => { }); }); it('should delete bucket policy', done => { - async.series([ - next => bucketPutPolicy(authInfo, _makeRequest(true), log, next), - next => bucketDeletePolicy(authInfo, _makeRequest(), log, next), - // eslint-disable-next-line no-unused-vars - next => metadata.getBucket(bucketName, log, (err, bucket, raftSessionId) => next(err, bucket)), - ], (err, results) => { - assert.equal(err, null, `Expected success, got error: ${err}`); - const bucket = results[2]; - const bucketPolicy = bucket.getBucketPolicy(); - assert.equal(bucketPolicy, null); - done(); - }); + async.series( + [ + next => bucketPutPolicy(authInfo, _makeRequest(true), log, next), + next => bucketDeletePolicy(authInfo, _makeRequest(), log, next), + // eslint-disable-next-line no-unused-vars + next => metadata.getBucket(bucketName, log, (err, bucket, raftSessionId) => next(err, bucket)), + ], + (err, results) => { + assert.equal(err, null, `Expected success, got error: ${err}`); + const bucket = results[2]; + const bucketPolicy = bucket.getBucketPolicy(); + assert.equal(bucketPolicy, null); + done(); + }, + ); }); }); diff --git a/tests/unit/api/bucketDeleteTagging.js b/tests/unit/api/bucketDeleteTagging.js index 7eeb98f1f8..73f17bf539 100644 --- a/tests/unit/api/bucketDeleteTagging.js +++ b/tests/unit/api/bucketDeleteTagging.js @@ -1,11 +1,7 @@ const assert = require('assert'); const { bucketPut } = require('../../../lib/api/bucketPut'); -const { cleanup, - DummyRequestLogger, - makeAuthInfo, - TaggingConfigTester } - = require('../helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo, TaggingConfigTester } = require('../helpers'); const bucketPutTagging = require('../../../lib/api/bucketPutTagging'); const bucketGetTagging = require('../../../lib/api/bucketGetTagging'); const bucketDeleteTagging = require('../../../lib/api/bucketDeleteTagging'); @@ -30,45 +26,41 @@ describe('deleteBucketTagging API', () => { it('should delete tags resource', done => { const taggingUtil = new TaggingConfigTester(); - const testBucketPutTaggingRequest = taggingUtil - .createBucketTaggingRequest('PUT', bucketName); + const testBucketPutTaggingRequest = taggingUtil.createBucketTaggingRequest('PUT', bucketName); bucketPutTagging(authInfo, testBucketPutTaggingRequest, log, err => { assert.strictEqual(err, undefined); - const testBucketGetTaggingRequest = taggingUtil - .createBucketTaggingRequest('GET', bucketName); - return bucketGetTagging(authInfo, testBucketGetTaggingRequest, log, - (err, xml) => { + const testBucketGetTaggingRequest = taggingUtil.createBucketTaggingRequest('GET', bucketName); + return bucketGetTagging(authInfo, testBucketGetTaggingRequest, log, (err, xml) => { + assert.ifError(err); + assert.strictEqual(xml, taggingUtil.constructXml()); + const testBucketDeleteTaggingRequest = taggingUtil.createBucketTaggingRequest('DELETE', bucketName); + return bucketDeleteTagging(authInfo, testBucketDeleteTaggingRequest, log, err => { assert.ifError(err); - assert.strictEqual(xml, taggingUtil.constructXml()); - const testBucketDeleteTaggingRequest = taggingUtil - .createBucketTaggingRequest('DELETE', bucketName); - return bucketDeleteTagging(authInfo, testBucketDeleteTaggingRequest, - log, err => { - assert.ifError(err); - return bucketGetTagging(authInfo, testBucketGetTaggingRequest, - log, err => { - assert(err.NoSuchTagSet); - return done(); - }); - }); + return bucketGetTagging(authInfo, testBucketGetTaggingRequest, log, err => { + assert(err.NoSuchTagSet); + return done(); + }); }); + }); }); }); it('should return access denied if the authorization check fails', done => { const taggingUtil = new TaggingConfigTester(); - const testBucketPutTaggingRequest = taggingUtil - .createBucketTaggingRequest('PUT', bucketName); + const testBucketPutTaggingRequest = taggingUtil.createBucketTaggingRequest('PUT', bucketName); bucketPutTagging(authInfo, testBucketPutTaggingRequest, log, err => { assert.ifError(err); - const testBucketDeleteTaggingRequest = taggingUtil - .createBucketTaggingRequest('DELETE', bucketName, null, true); - return bucketDeleteTagging(authInfo, testBucketDeleteTaggingRequest, - log, err => { - assert(err.AccessDenied); - return done(); - }); + const testBucketDeleteTaggingRequest = taggingUtil.createBucketTaggingRequest( + 'DELETE', + bucketName, + null, + true, + ); + return bucketDeleteTagging(authInfo, testBucketDeleteTaggingRequest, log, err => { + assert(err.AccessDenied); + return done(); + }); }); }); }); diff --git a/tests/unit/api/bucketDeleteWebsite.js b/tests/unit/api/bucketDeleteWebsite.js index 41bd286c87..4d1623e363 100644 --- a/tests/unit/api/bucketDeleteWebsite.js +++ b/tests/unit/api/bucketDeleteWebsite.js @@ -3,19 +3,14 @@ const assert = require('assert'); const { bucketPut } = require('../../../lib/api/bucketPut'); const bucketPutWebsite = require('../../../lib/api/bucketPutWebsite'); const bucketDeleteWebsite = require('../../../lib/api/bucketDeleteWebsite'); -const { cleanup, - DummyRequestLogger, - makeAuthInfo, - WebsiteConfig } -= require('../helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo, WebsiteConfig } = require('../helpers'); const metadata = require('../../../lib/metadata/wrapper'); const log = new DummyRequestLogger(); const authInfo = makeAuthInfo('accessKey1'); const bucketName = 'bucketname'; const config = new WebsiteConfig('index.html', 'error.html'); -config.addRoutingRule({ ReplaceKeyPrefixWith: 'documents/' }, -{ KeyPrefixEquals: 'docs/' }); +config.addRoutingRule({ ReplaceKeyPrefixWith: 'documents/' }, { KeyPrefixEquals: 'docs/' }); const testBucketPutRequest = { bucketName, headers: { host: `${bucketName}.s3.amazonaws.com` }, @@ -31,8 +26,7 @@ const testBucketDeleteWebsiteRequest = { query: { website: '' }, actionImplicitDenies: false, }; -const testBucketPutWebsiteRequest = Object.assign({ post: config.getXml() }, - testBucketDeleteWebsiteRequest); +const testBucketPutWebsiteRequest = Object.assign({ post: config.getXml() }, testBucketDeleteWebsiteRequest); describe('deleteBucketWebsite API', () => { beforeEach(done => { @@ -43,9 +37,8 @@ describe('deleteBucketWebsite API', () => { }); afterEach(() => cleanup()); - it('should delete a bucket\'s website configuration in metadata', done => { - bucketDeleteWebsite(authInfo, testBucketDeleteWebsiteRequest, log, - err => { + it("should delete a bucket's website configuration in metadata", done => { + bucketDeleteWebsite(authInfo, testBucketDeleteWebsiteRequest, log, err => { if (err) { process.stdout.write(`Unexpected err ${err}`); return done(err); @@ -55,8 +48,7 @@ describe('deleteBucketWebsite API', () => { process.stdout.write(`Err retrieving bucket MD ${err}`); return done(err); } - assert.strictEqual(bucket.getWebsiteConfiguration(), - null); + assert.strictEqual(bucket.getWebsiteConfiguration(), null); return done(); }); }); diff --git a/tests/unit/api/bucketGet.js b/tests/unit/api/bucketGet.js index 7968ca78a1..61fcbc4565 100644 --- a/tests/unit/api/bucketGet.js +++ b/tests/unit/api/bucketGet.js @@ -24,40 +24,55 @@ const objectName1 = `${prefix}${delimiter}objectName1`; const objectName2 = `${prefix}${delimiter}objectName2`; const objectName3 = 'invalidURI~~~b'; const objectName4 = `${objectName1}&><"\'`; -const testPutBucketRequest = new DummyRequest({ - bucketName, - headers: {}, - url: `/${bucketName}`, - namespace, -}, Buffer.alloc(0)); -const testPutObjectRequest1 = new DummyRequest({ - bucketName, - headers: {}, - url: `/${bucketName}/${objectName1}`, - namespace, - objectKey: objectName1, -}, postBody); -const testPutObjectRequest2 = new DummyRequest({ - bucketName, - headers: {}, - url: `/${bucketName}/${objectName2}`, - namespace, - objectKey: objectName2, -}, postBody); -const testPutObjectRequest3 = new DummyRequest({ - bucketName, - headers: {}, - url: `/${bucketName}/${objectName3}`, - namespace, - objectKey: objectName3, -}, postBody); -const testPutObjectRequest4 = new DummyRequest({ - bucketName, - headers: {}, - url: `/${bucketName}/${objectName3}`, - namespace, - objectKey: objectName4, -}, postBody); +const testPutBucketRequest = new DummyRequest( + { + bucketName, + headers: {}, + url: `/${bucketName}`, + namespace, + }, + Buffer.alloc(0), +); +const testPutObjectRequest1 = new DummyRequest( + { + bucketName, + headers: {}, + url: `/${bucketName}/${objectName1}`, + namespace, + objectKey: objectName1, + }, + postBody, +); +const testPutObjectRequest2 = new DummyRequest( + { + bucketName, + headers: {}, + url: `/${bucketName}/${objectName2}`, + namespace, + objectKey: objectName2, + }, + postBody, +); +const testPutObjectRequest3 = new DummyRequest( + { + bucketName, + headers: {}, + url: `/${bucketName}/${objectName3}`, + namespace, + objectKey: objectName3, + }, + postBody, +); +const testPutObjectRequest4 = new DummyRequest( + { + bucketName, + headers: {}, + url: `/${bucketName}/${objectName3}`, + namespace, + objectKey: objectName4, + }, + postBody, +); const baseGetRequest = { bucketName, @@ -72,39 +87,33 @@ const tests = [ name: 'list of all objects if no delimiter specified', request: Object.assign({ query: {}, url: baseUrl }, baseGetRequest), assertion: result => { - assert.strictEqual(result.ListBucketResult.Contents[1].Key[0], - objectName1); - assert.strictEqual(result.ListBucketResult.Contents[2].Key[0], - objectName2); - assert.strictEqual(result.ListBucketResult.Contents[0].Key[0], - objectName3); + assert.strictEqual(result.ListBucketResult.Contents[1].Key[0], objectName1); + assert.strictEqual(result.ListBucketResult.Contents[2].Key[0], objectName2); + assert.strictEqual(result.ListBucketResult.Contents[0].Key[0], objectName3); }, }, { - name: 'return name of common prefix of common prefix objects if ' + - 'delimiter and prefix specified', - request: Object.assign({ - url: `/${bucketName}?delimiter=${delimiter}&prefix=${prefix}`, - query: { delimiter, prefix }, - }, baseGetRequest), + name: 'return name of common prefix of common prefix objects if ' + 'delimiter and prefix specified', + request: Object.assign( + { + url: `/${bucketName}?delimiter=${delimiter}&prefix=${prefix}`, + query: { delimiter, prefix }, + }, + baseGetRequest, + ), assertion: result => - assert.strictEqual(result.ListBucketResult - .CommonPrefixes[0].Prefix[0], `${prefix}${delimiter}`), + assert.strictEqual(result.ListBucketResult.CommonPrefixes[0].Prefix[0], `${prefix}${delimiter}`), }, { name: 'return empty list when max-keys is set to 0', - request: Object.assign({ query: { 'max-keys': '0' }, url: baseUrl }, - baseGetRequest), - assertion: result => - assert.strictEqual(result.ListBucketResult.Contents, undefined), + request: Object.assign({ query: { 'max-keys': '0' }, url: baseUrl }, baseGetRequest), + assertion: result => assert.strictEqual(result.ListBucketResult.Contents, undefined), }, { name: 'return no more keys than max-keys specified', - request: Object.assign({ query: { 'max-keys': '1' }, url: baseUrl }, - baseGetRequest), + request: Object.assign({ query: { 'max-keys': '1' }, url: baseUrl }, baseGetRequest), assertion: result => { - assert.strictEqual(result.ListBucketResult.Contents[0].Key[0], - objectName3); + assert.strictEqual(result.ListBucketResult.Contents[0].Key[0], objectName3); assert.strictEqual(result.ListBucketResult.Contents[1], undefined); }, }, @@ -115,16 +124,12 @@ const tests = [ query: { 'max-keys': '1' }, url: baseUrl, }, - baseGetRequest + baseGetRequest, ), assertion: result => { - assert.strictEqual(result.ListBucketResult.Contents[0].Key[0], - objectName3); + assert.strictEqual(result.ListBucketResult.Contents[0].Key[0], objectName3); assert.strictEqual(result.ListBucketResult.Contents[1], undefined); - assert.strictEqual( - result.ListBucketResult.NextContinuationToken[0], - 'aW52YWxpZFVSSX5+fmI=' - ); + assert.strictEqual(result.ListBucketResult.NextContinuationToken[0], 'aW52YWxpZFVSSX5+fmI='); }, }, { @@ -134,43 +139,30 @@ const tests = [ query: { 'encoding-type': 'url', 'max-keys': '1' }, url: baseUrl, }, - baseGetRequest + baseGetRequest, ), assertion: result => { - assert.strictEqual(result.ListBucketResult.Contents[0].Key[0], - objectName3); + assert.strictEqual(result.ListBucketResult.Contents[0].Key[0], objectName3); assert.strictEqual(result.ListBucketResult.Contents[1], undefined); - assert.strictEqual( - result.ListBucketResult.NextContinuationToken[0], - 'aW52YWxpZFVSSX5+fmI=' - ); + assert.strictEqual(result.ListBucketResult.NextContinuationToken[0], 'aW52YWxpZFVSSX5+fmI='); }, }, { - name: 'return max-keys number from request even if greater than ' + - 'actual keys returned', - request: Object.assign({ query: { 'max-keys': '99999' }, url: baseUrl }, - baseGetRequest), - assertion: result => - assert.strictEqual(result.ListBucketResult.MaxKeys[0], '99999'), + name: 'return max-keys number from request even if greater than ' + 'actual keys returned', + request: Object.assign({ query: { 'max-keys': '99999' }, url: baseUrl }, baseGetRequest), + assertion: result => assert.strictEqual(result.ListBucketResult.MaxKeys[0], '99999'), }, { name: 'return max-keys number from request even when value is 0', - request: Object.assign({ query: { 'max-keys': '0' }, url: baseUrl }, - baseGetRequest), - assertion: result => - assert.strictEqual(result.ListBucketResult.MaxKeys[0], '0'), + request: Object.assign({ query: { 'max-keys': '0' }, url: baseUrl }, baseGetRequest), + assertion: result => assert.strictEqual(result.ListBucketResult.MaxKeys[0], '0'), }, { name: 'url encode object key name if requested', - request: Object.assign( - { query: { 'encoding-type': 'url' }, url: baseUrl }, - baseGetRequest), + request: Object.assign({ query: { 'encoding-type': 'url' }, url: baseUrl }, baseGetRequest), assertion: result => { - assert.strictEqual(result.ListBucketResult.Contents[0].Key[0], - querystring.escape(objectName3)); - assert.strictEqual(result.ListBucketResult.Contents[1].Key[0], - querystring.escape(objectName1)); + assert.strictEqual(result.ListBucketResult.Contents[0].Key[0], querystring.escape(objectName3)); + assert.strictEqual(result.ListBucketResult.Contents[1].Key[0], querystring.escape(objectName1)); }, }, ]; @@ -184,28 +176,25 @@ describe('bucketGet API', () => { it(`should ${test.name}`, done => { const testGetRequest = test.request; - async.waterfall([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - (_, next) => objectPut(authInfo, - testPutObjectRequest1, undefined, log, next), - (_, next) => objectPut(authInfo, - testPutObjectRequest2, undefined, log, next), - (_, next) => objectPut(authInfo, - testPutObjectRequest3, undefined, log, next), - (_, next) => - bucketGet(authInfo, testGetRequest, log, next), - (result, _, next) => parseString(result, next), - ], - (_, result) => { - test.assertion(result); - done(); - }); + async.waterfall( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + (_, next) => objectPut(authInfo, testPutObjectRequest1, undefined, log, next), + (_, next) => objectPut(authInfo, testPutObjectRequest2, undefined, log, next), + (_, next) => objectPut(authInfo, testPutObjectRequest3, undefined, log, next), + (_, next) => bucketGet(authInfo, testGetRequest, log, next), + (result, _, next) => parseString(result, next), + ], + (_, result) => { + test.assertion(result); + done(); + }, + ); }); }); it('should return an InvalidArgument error if max-keys == -1', done => { - const testGetRequest = Object.assign({ query: { 'max-keys': '-1' } }, - baseGetRequest); + const testGetRequest = Object.assign({ query: { 'max-keys': '-1' } }, baseGetRequest); bucketGet(authInfo, testGetRequest, log, err => { assert.strictEqual(err.is.InvalidArgument, true); done(); @@ -213,81 +202,73 @@ describe('bucketGet API', () => { }); it('should escape invalid xml characters in object key names', done => { - const testGetRequest = Object.assign({ query: {}, url: baseUrl }, - baseGetRequest); - - async.waterfall([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - (_, next) => objectPut(authInfo, testPutObjectRequest4, - undefined, log, next), - (_, next) => bucketGet(authInfo, testGetRequest, - log, next), - (result, _, next) => parseString(result, next), - ], - (_, result) => { - assert.strictEqual(result.ListBucketResult.Contents[0].Key[0], - testPutObjectRequest4.objectKey); - done(); - }); + const testGetRequest = Object.assign({ query: {}, url: baseUrl }, baseGetRequest); + + async.waterfall( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + (_, next) => objectPut(authInfo, testPutObjectRequest4, undefined, log, next), + (_, next) => bucketGet(authInfo, testGetRequest, log, next), + (result, _, next) => parseString(result, next), + ], + (_, result) => { + assert.strictEqual(result.ListBucketResult.Contents[0].Key[0], testPutObjectRequest4.objectKey); + done(); + }, + ); }); it('should return xml that refers to the s3 docs for xml specs', done => { - const testGetRequest = Object.assign({ query: {}, url: baseUrl }, - baseGetRequest); - - async.waterfall([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - (_, next) => - bucketGet(authInfo, testGetRequest, log, next), - (result, _, next) => parseString(result, next), - ], - (_, result) => { - assert.strictEqual(result.ListBucketResult.$.xmlns, - 'http://s3.amazonaws.com/doc/2006-03-01/'); - done(); - }); + const testGetRequest = Object.assign({ query: {}, url: baseUrl }, baseGetRequest); + + async.waterfall( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + (_, next) => bucketGet(authInfo, testGetRequest, log, next), + (result, _, next) => parseString(result, next), + ], + (_, result) => { + assert.strictEqual(result.ListBucketResult.$.xmlns, 'http://s3.amazonaws.com/doc/2006-03-01/'); + done(); + }, + ); }); }); -const testsForV2 = [...tests, +const testsForV2 = [ + ...tests, { name: 'return no owner info when --fetch-owner option is not used', request: Object.assign({ query: {}, url: baseUrl }, baseGetRequest), assertion: result => { - const owners - = result.ListBucketResult.Contents.filter(c => c.Owner); + const owners = result.ListBucketResult.Contents.filter(c => c.Owner); assert.strictEqual(owners.length, 0); }, }, { name: 'return owner info when --fetch-owner option is used', - request: Object.assign({ query: { 'fetch-owner': 'true' }, - url: baseUrl }, baseGetRequest), + request: Object.assign({ query: { 'fetch-owner': 'true' }, url: baseUrl }, baseGetRequest), assertion: result => { - const owners - = result.ListBucketResult.Contents.filter(c => + const owners = result.ListBucketResult.Contents.filter( + c => c.Owner[0].ID[0] === authInfo.canonicalID && - c.Owner[0].DisplayName[0] === authInfo.accountDisplayName); - assert.strictEqual(owners.length, - result.ListBucketResult.Contents.length); + c.Owner[0].DisplayName[0] === authInfo.accountDisplayName, + ); + assert.strictEqual(owners.length, result.ListBucketResult.Contents.length); }, }, { name: 'return no owner info when --no-fetch-owner option is used', - request: Object.assign({ query: { 'fetch-owner': 'false' }, - url: baseUrl }, baseGetRequest), + request: Object.assign({ query: { 'fetch-owner': 'false' }, url: baseUrl }, baseGetRequest), assertion: result => { - const owners - = result.ListBucketResult.Contents.filter(c => c.Owner); + const owners = result.ListBucketResult.Contents.filter(c => c.Owner); assert.strictEqual(owners.length, 0); }, }, { name: 'return max-keys number from request even when value is 0', - request: Object.assign({ query: { 'max-keys': '0' }, url: baseUrl }, - baseGetRequest), - assertion: result => - assert.strictEqual(result.ListBucketResult.MaxKeys[0], '0'), + request: Object.assign({ query: { 'max-keys': '0' }, url: baseUrl }, baseGetRequest), + assertion: result => assert.strictEqual(result.ListBucketResult.MaxKeys[0], '0'), }, ]; @@ -299,286 +280,350 @@ describe('bucketGet API V2', () => { testsForV2.forEach(test => { /* eslint-disable no-param-reassign */ test.request.query['list-type'] = 2; - test.request.url = test.request.url.indexOf('?') > -1 ? - `${test.request.url}&list-type=2` : - `${test.request.url}?list-type=2`; + test.request.url = + test.request.url.indexOf('?') > -1 ? `${test.request.url}&list-type=2` : `${test.request.url}?list-type=2`; /* eslint-enable no-param-reassign */ it(`should ${test.name}`, done => { const testGetRequest = test.request; - async.waterfall([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - (_, next) => objectPut(authInfo, - testPutObjectRequest1, undefined, log, next), - (_, next) => objectPut(authInfo, - testPutObjectRequest2, undefined, log, next), - (_, next) => objectPut(authInfo, - testPutObjectRequest3, undefined, log, next), - (_, next) => - bucketGet(authInfo, testGetRequest, log, next), - (result, _, next) => parseString(result, next), - ], - (_, result) => { - // v2 requests should return 'KeyCount' in response - const keyCount = - Number.parseInt(result.ListBucketResult.KeyCount[0], 10); - const keysReturned = result.ListBucketResult.Contents ? - result.ListBucketResult.Contents.length : 0; - assert.strictEqual(keyCount, keysReturned); - // assert the results from tests - test.assertion(result); - if (result.ListBucketResult.IsTruncated && result.ListBucketResult.IsTruncated[0] === 'false') { - assert.strictEqual(result.ListBucketResult.NextContinuationToken, undefined); - } - done(); - }); + async.waterfall( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + (_, next) => objectPut(authInfo, testPutObjectRequest1, undefined, log, next), + (_, next) => objectPut(authInfo, testPutObjectRequest2, undefined, log, next), + (_, next) => objectPut(authInfo, testPutObjectRequest3, undefined, log, next), + (_, next) => bucketGet(authInfo, testGetRequest, log, next), + (result, _, next) => parseString(result, next), + ], + (_, result) => { + // v2 requests should return 'KeyCount' in response + const keyCount = Number.parseInt(result.ListBucketResult.KeyCount[0], 10); + const keysReturned = result.ListBucketResult.Contents ? result.ListBucketResult.Contents.length : 0; + assert.strictEqual(keyCount, keysReturned); + // assert the results from tests + test.assertion(result); + if (result.ListBucketResult.IsTruncated && result.ListBucketResult.IsTruncated[0] === 'false') { + assert.strictEqual(result.ListBucketResult.NextContinuationToken, undefined); + } + done(); + }, + ); }); }); describe('x-amz-optional-object-attributes header', () => { it('should return an error if the header is empty', done => { - const testGetRequest = Object.assign({ - query: {}, - url: baseUrl, - }, baseGetRequest); + const testGetRequest = Object.assign( + { + query: {}, + url: baseUrl, + }, + baseGetRequest, + ); testGetRequest.headers['x-amz-optional-object-attributes'] = ''; - async.waterfall([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - (_, next) => bucketGet(authInfo, testGetRequest, log, next), - (result, _, next) => parseString(result, next), - ], err => { - assert.strictEqual(err.is.InvalidArgument, true); - done(); - }); + async.waterfall( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + (_, next) => bucketGet(authInfo, testGetRequest, log, next), + (result, _, next) => parseString(result, next), + ], + err => { + assert.strictEqual(err.is.InvalidArgument, true); + done(); + }, + ); }); it('should return an error for invalid optional attributes', done => { - const testGetRequest = Object.assign({ - query: {}, - url: baseUrl, - }, baseGetRequest); + const testGetRequest = Object.assign( + { + query: {}, + url: baseUrl, + }, + baseGetRequest, + ); testGetRequest.headers['x-amz-optional-object-attributes'] = 'InvalidAttribute'; - async.series([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - next => bucketGet(authInfo, testGetRequest, log, next), - ], err => { - assert.strictEqual(err.is.InvalidArgument, true); - done(); - }); + async.series( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + next => bucketGet(authInfo, testGetRequest, log, next), + ], + err => { + assert.strictEqual(err.is.InvalidArgument, true); + done(); + }, + ); }); it('should accept wildcard value', done => { - const testGetRequest = Object.assign({ - query: {}, - url: baseUrl, - }, baseGetRequest); + const testGetRequest = Object.assign( + { + query: {}, + url: baseUrl, + }, + baseGetRequest, + ); testGetRequest.headers['x-amz-optional-object-attributes'] = 'x-amz-meta-*'; - async.waterfall([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - (_, next) => bucketGet(authInfo, testGetRequest, log, next), - (result, _, next) => parseString(result, next), - ], (err, result) => { - assert.strictEqual(err, null); - assert.strictEqual(result.ListBucketResult.$.xmlns, 'http://s3.amazonaws.com/doc/2006-03-01/'); - done(); - }); + async.waterfall( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + (_, next) => bucketGet(authInfo, testGetRequest, log, next), + (result, _, next) => parseString(result, next), + ], + (err, result) => { + assert.strictEqual(err, null); + assert.strictEqual(result.ListBucketResult.$.xmlns, 'http://s3.amazonaws.com/doc/2006-03-01/'); + done(); + }, + ); }); it('should return an error for a mix of valid and invalid attributes', done => { - const testGetRequest = Object.assign({ - query: {}, - url: baseUrl, - }, baseGetRequest); + const testGetRequest = Object.assign( + { + query: {}, + url: baseUrl, + }, + baseGetRequest, + ); testGetRequest.headers['x-amz-optional-object-attributes'] = 'RestoreStatus,InvalidAttribute'; - async.series([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - next => bucketGet(authInfo, testGetRequest, log, next), - ], err => { - assert.strictEqual(err.is.InvalidArgument, true); - done(); - }); + async.series( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + next => bucketGet(authInfo, testGetRequest, log, next), + ], + err => { + assert.strictEqual(err.is.InvalidArgument, true); + done(); + }, + ); }); it('should handle attributes with leading/trailing whitespace', done => { - const testGetRequest = Object.assign({ - query: {}, - url: baseUrl, - }, baseGetRequest); + const testGetRequest = Object.assign( + { + query: {}, + url: baseUrl, + }, + baseGetRequest, + ); testGetRequest.headers['x-amz-optional-object-attributes'] = ' x-amz-meta-foo '; - async.waterfall([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - (_, next) => bucketGet(authInfo, testGetRequest, log, next), - (result, _, next) => parseString(result, next), - ], - (err, result) => { - assert.strictEqual(err, null); - assert.strictEqual(result.ListBucketResult.$.xmlns, 'http://s3.amazonaws.com/doc/2006-03-01/'); - done(); - }); + async.waterfall( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + (_, next) => bucketGet(authInfo, testGetRequest, log, next), + (result, _, next) => parseString(result, next), + ], + (err, result) => { + assert.strictEqual(err, null); + assert.strictEqual(result.ListBucketResult.$.xmlns, 'http://s3.amazonaws.com/doc/2006-03-01/'); + done(); + }, + ); }); it('should handle multiple valid attributes', done => { - const testGetRequest = Object.assign({ - query: {}, - url: baseUrl, - }, baseGetRequest); + const testGetRequest = Object.assign( + { + query: {}, + url: baseUrl, + }, + baseGetRequest, + ); testGetRequest.headers['x-amz-optional-object-attributes'] = 'RestoreStatus,x-amz-meta-foo,x-amz-meta-bar'; - async.waterfall([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - (_, next) => bucketGet(authInfo, testGetRequest, log, next), - (result, _, next) => parseString(result, next), - ], - (err, result) => { - assert.strictEqual(err, null); - assert.strictEqual(result.ListBucketResult.$.xmlns, 'http://s3.amazonaws.com/doc/2006-03-01/'); - done(); - }); + async.waterfall( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + (_, next) => bucketGet(authInfo, testGetRequest, log, next), + (result, _, next) => parseString(result, next), + ], + (err, result) => { + assert.strictEqual(err, null); + assert.strictEqual(result.ListBucketResult.$.xmlns, 'http://s3.amazonaws.com/doc/2006-03-01/'); + done(); + }, + ); }); it('should return user metadata if requested and present', done => { const objectNameMeta = 'objectWithMeta'; - const putRequest = new DummyRequest({ - bucketName, - headers: { 'x-amz-meta-color': 'red' }, - url: `/${bucketName}/${objectNameMeta}`, - namespace, - objectKey: objectNameMeta, - }, postBody); + const putRequest = new DummyRequest( + { + bucketName, + headers: { 'x-amz-meta-color': 'red' }, + url: `/${bucketName}/${objectNameMeta}`, + namespace, + objectKey: objectNameMeta, + }, + postBody, + ); - const testGetRequest = Object.assign({ - query: {}, - url: baseUrl, - }, baseGetRequest); + const testGetRequest = Object.assign( + { + query: {}, + url: baseUrl, + }, + baseGetRequest, + ); testGetRequest.headers['x-amz-optional-object-attributes'] = 'x-amz-meta-color'; - async.waterfall([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - (_, next) => objectPut(authInfo, putRequest, undefined, log, next), - (_, next) => bucketGet(authInfo, testGetRequest, log, next), - (result, _, next) => parseString(result, next), - ], - (err, result) => { - assert.strictEqual(err, null); - const content = result.ListBucketResult.Contents[0]; - assert.strictEqual(content.Key[0], objectNameMeta); - assert.strictEqual(content['x-amz-meta-color'][0], 'red'); - done(); - }); + async.waterfall( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + (_, next) => objectPut(authInfo, putRequest, undefined, log, next), + (_, next) => bucketGet(authInfo, testGetRequest, log, next), + (result, _, next) => parseString(result, next), + ], + (err, result) => { + assert.strictEqual(err, null); + const content = result.ListBucketResult.Contents[0]; + assert.strictEqual(content.Key[0], objectNameMeta); + assert.strictEqual(content['x-amz-meta-color'][0], 'red'); + done(); + }, + ); }); it('should not duplicate elements when the header repeats tokens', done => { const objectNameMeta = 'objectWithRepeatedTokens'; - const putRequest = new DummyRequest({ - bucketName, - headers: { 'x-amz-meta-color': 'red' }, - url: `/${bucketName}/${objectNameMeta}`, - namespace, - objectKey: objectNameMeta, - }, postBody); + const putRequest = new DummyRequest( + { + bucketName, + headers: { 'x-amz-meta-color': 'red' }, + url: `/${bucketName}/${objectNameMeta}`, + namespace, + objectKey: objectNameMeta, + }, + postBody, + ); - const testGetRequest = Object.assign({ - query: {}, - url: baseUrl, - }, baseGetRequest); + const testGetRequest = Object.assign( + { + query: {}, + url: baseUrl, + }, + baseGetRequest, + ); testGetRequest.headers['x-amz-optional-object-attributes'] = 'RestoreStatus,RestoreStatus,x-amz-meta-color,x-amz-meta-color'; - async.waterfall([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - (_, next) => objectPut(authInfo, putRequest, undefined, log, next), - (_, next) => bucketGet(authInfo, testGetRequest, log, next), - (result, _, next) => parseString(result, next), - ], - (err, result) => { - assert.strictEqual(err, null); - const content = result.ListBucketResult.Contents[0]; - assert.strictEqual(content.Key[0], objectNameMeta); - assert.strictEqual(content.RestoreStatus.length, 1); - assert.strictEqual(content['x-amz-meta-color'].length, 1); - assert.strictEqual(content['x-amz-meta-color'][0], 'red'); - done(); - }); + async.waterfall( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + (_, next) => objectPut(authInfo, putRequest, undefined, log, next), + (_, next) => bucketGet(authInfo, testGetRequest, log, next), + (result, _, next) => parseString(result, next), + ], + (err, result) => { + assert.strictEqual(err, null); + const content = result.ListBucketResult.Contents[0]; + assert.strictEqual(content.Key[0], objectNameMeta); + assert.strictEqual(content.RestoreStatus.length, 1); + assert.strictEqual(content['x-amz-meta-color'].length, 1); + assert.strictEqual(content['x-amz-meta-color'][0], 'red'); + done(); + }, + ); }); it('should return all user metadata if wildcard requested', done => { const objectNameMeta = 'objectWithMetaWildcard'; - const putRequest = new DummyRequest({ - bucketName, - headers: { 'x-amz-meta-color': 'red', 'x-amz-meta-size': 'large' }, - url: `/${bucketName}/${objectNameMeta}`, - namespace, - objectKey: objectNameMeta, - }, postBody); + const putRequest = new DummyRequest( + { + bucketName, + headers: { 'x-amz-meta-color': 'red', 'x-amz-meta-size': 'large' }, + url: `/${bucketName}/${objectNameMeta}`, + namespace, + objectKey: objectNameMeta, + }, + postBody, + ); - const testGetRequest = Object.assign({ - query: {}, - url: baseUrl, - }, baseGetRequest); + const testGetRequest = Object.assign( + { + query: {}, + url: baseUrl, + }, + baseGetRequest, + ); testGetRequest.headers['x-amz-optional-object-attributes'] = 'x-amz-meta-*'; - async.waterfall([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - (_, next) => objectPut(authInfo, putRequest, undefined, log, next), - (_, next) => bucketGet(authInfo, testGetRequest, log, next), - (result, _, next) => parseString(result, next), - ], - (err, result) => { - assert.strictEqual(err, null); - const content = result.ListBucketResult.Contents[0]; - assert.strictEqual(content.Key[0], objectNameMeta); - assert.strictEqual(content['x-amz-meta-color'][0], 'red'); - assert.strictEqual(content['x-amz-meta-size'][0], 'large'); - done(); - }); + async.waterfall( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + (_, next) => objectPut(authInfo, putRequest, undefined, log, next), + (_, next) => bucketGet(authInfo, testGetRequest, log, next), + (result, _, next) => parseString(result, next), + ], + (err, result) => { + assert.strictEqual(err, null); + const content = result.ListBucketResult.Contents[0]; + assert.strictEqual(content.Key[0], objectNameMeta); + assert.strictEqual(content['x-amz-meta-color'][0], 'red'); + assert.strictEqual(content['x-amz-meta-size'][0], 'large'); + done(); + }, + ); }); it('should return user metadata in version listing if requested', done => { const objectNameMeta = 'objectWithMetaVersion'; - const putRequest = new DummyRequest({ - bucketName, - headers: { 'x-amz-meta-ver': '1' }, - url: `/${bucketName}/${objectNameMeta}`, - namespace, - objectKey: objectNameMeta, - }, postBody); + const putRequest = new DummyRequest( + { + bucketName, + headers: { 'x-amz-meta-ver': '1' }, + url: `/${bucketName}/${objectNameMeta}`, + namespace, + objectKey: objectNameMeta, + }, + postBody, + ); - const testGetRequest = Object.assign({ - query: { versions: '' }, - url: `${baseUrl}?versions`, - }, baseGetRequest); + const testGetRequest = Object.assign( + { + query: { versions: '' }, + url: `${baseUrl}?versions`, + }, + baseGetRequest, + ); testGetRequest.headers['x-amz-optional-object-attributes'] = 'x-amz-meta-ver'; - async.waterfall([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - (_, next) => objectPut(authInfo, putRequest, undefined, log, next), - (_, next) => bucketGet(authInfo, testGetRequest, log, next), - (result, _, next) => parseString(result, next), - ], - (err, result) => { - assert.strictEqual(err, null); - const version = result.ListVersionsResult.Version[0]; - assert.strictEqual(version.Key[0], objectNameMeta); - assert.strictEqual(version['x-amz-meta-ver'][0], '1'); - done(); - }); + async.waterfall( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + (_, next) => objectPut(authInfo, putRequest, undefined, log, next), + (_, next) => bucketGet(authInfo, testGetRequest, log, next), + (result, _, next) => parseString(result, next), + ], + (err, result) => { + assert.strictEqual(err, null); + const version = result.ListVersionsResult.Version[0]; + assert.strictEqual(version.Key[0], objectNameMeta); + assert.strictEqual(version['x-amz-meta-ver'][0], '1'); + done(); + }, + ); }); it('should not include optional attributes on delete markers in versions listing', done => { const objectNameMeta = 'objectWithMetaAndDeleteMarker'; - const putRequest = new DummyRequest({ - bucketName, - headers: { 'x-amz-meta-color': 'red' }, - url: `/${bucketName}/${objectNameMeta}`, - namespace, - objectKey: objectNameMeta, - }, postBody); + const putRequest = new DummyRequest( + { + bucketName, + headers: { 'x-amz-meta-color': 'red' }, + url: `/${bucketName}/${objectNameMeta}`, + namespace, + objectKey: objectNameMeta, + }, + postBody, + ); const deleteRequest = new DummyRequest({ bucketName, headers: {}, @@ -593,70 +638,83 @@ describe('bucketGet API V2', () => { url: '/?versioning', query: { versioning: '' }, actionImplicitDenies: false, - post: '' - + 'Enabled', + post: + '' + + 'Enabled', }; - const testGetRequest = Object.assign({ - query: { versions: '' }, - url: `${baseUrl}?versions`, - }, baseGetRequest); - testGetRequest.headers['x-amz-optional-object-attributes'] = - 'RestoreStatus,x-amz-meta-color'; - - async.waterfall([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - (_, next) => bucketPutVersioning(authInfo, versioningRequest, log, next), - (_, next) => objectPut(authInfo, putRequest, undefined, log, next), - (_, next) => objectDelete(authInfo, deleteRequest, log, next), - (_, next) => bucketGet(authInfo, testGetRequest, log, next), - (result, _, next) => parseString(result, next), - ], - (err, result) => { - assert.strictEqual(err, null); - const { Version, DeleteMarker } = result.ListVersionsResult; - assert.strictEqual(Version.length, 1); - assert.strictEqual(Version[0].Key[0], objectNameMeta); - assert.strictEqual(Version[0]['x-amz-meta-color'][0], 'red'); - assert.strictEqual(Version[0].RestoreStatus.length, 1); - - assert.strictEqual(DeleteMarker.length, 1); - assert.strictEqual(DeleteMarker[0].Key[0], objectNameMeta); - assert.strictEqual(DeleteMarker[0]['x-amz-meta-color'], undefined); - assert.strictEqual(DeleteMarker[0].RestoreStatus, undefined); - done(); - }); + const testGetRequest = Object.assign( + { + query: { versions: '' }, + url: `${baseUrl}?versions`, + }, + baseGetRequest, + ); + testGetRequest.headers['x-amz-optional-object-attributes'] = 'RestoreStatus,x-amz-meta-color'; + + async.waterfall( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + (_, next) => bucketPutVersioning(authInfo, versioningRequest, log, next), + (_, next) => objectPut(authInfo, putRequest, undefined, log, next), + (_, next) => objectDelete(authInfo, deleteRequest, log, next), + (_, next) => bucketGet(authInfo, testGetRequest, log, next), + (result, _, next) => parseString(result, next), + ], + (err, result) => { + assert.strictEqual(err, null); + const { Version, DeleteMarker } = result.ListVersionsResult; + assert.strictEqual(Version.length, 1); + assert.strictEqual(Version[0].Key[0], objectNameMeta); + assert.strictEqual(Version[0]['x-amz-meta-color'][0], 'red'); + assert.strictEqual(Version[0].RestoreStatus.length, 1); + + assert.strictEqual(DeleteMarker.length, 1); + assert.strictEqual(DeleteMarker[0].Key[0], objectNameMeta); + assert.strictEqual(DeleteMarker[0]['x-amz-meta-color'], undefined); + assert.strictEqual(DeleteMarker[0].RestoreStatus, undefined); + done(); + }, + ); }); it('should return user metadata as case insentive (lowercase header)', done => { const objectNameMeta = 'objectWithMeta'; - const putRequest = new DummyRequest({ - bucketName, - headers: { 'x-amz-meta-color': 'yellow' }, - url: `/${bucketName}/${objectNameMeta}`, - namespace, - objectKey: objectNameMeta, - }, postBody); + const putRequest = new DummyRequest( + { + bucketName, + headers: { 'x-amz-meta-color': 'yellow' }, + url: `/${bucketName}/${objectNameMeta}`, + namespace, + objectKey: objectNameMeta, + }, + postBody, + ); - const testGetRequest = Object.assign({ - query: {}, - url: baseUrl, - }, baseGetRequest); + const testGetRequest = Object.assign( + { + query: {}, + url: baseUrl, + }, + baseGetRequest, + ); testGetRequest.headers['x-amz-optional-object-attributes'] = 'x-amz-meta-coLor'; - async.waterfall([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - (_, next) => objectPut(authInfo, putRequest, undefined, log, next), - (_, next) => bucketGet(authInfo, testGetRequest, log, next), - (result, _, next) => parseString(result, next), - ], - (err, result) => { - assert.strictEqual(err, null); - const content = result.ListBucketResult.Contents[0]; - assert.strictEqual(content.Key[0], objectNameMeta); - assert.strictEqual(content['x-amz-meta-color'][0], 'yellow'); - done(); - }); + async.waterfall( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + (_, next) => objectPut(authInfo, putRequest, undefined, log, next), + (_, next) => bucketGet(authInfo, testGetRequest, log, next), + (result, _, next) => parseString(result, next), + ], + (err, result) => { + assert.strictEqual(err, null); + const content = result.ListBucketResult.Contents[0]; + assert.strictEqual(content.Key[0], objectNameMeta); + assert.strictEqual(content['x-amz-meta-color'][0], 'yellow'); + done(); + }, + ); }); }); }); diff --git a/tests/unit/api/bucketGetACL.js b/tests/unit/api/bucketGetACL.js index 5a4327a9d7..3f013982a9 100644 --- a/tests/unit/api/bucketGetACL.js +++ b/tests/unit/api/bucketGetACL.js @@ -41,7 +41,7 @@ describe('bucketGetACL API', () => { bucketName, namespace, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-acl': 'private', }, url: '/?acl', @@ -49,24 +49,26 @@ describe('bucketGetACL API', () => { actionImplicitDenies: false, }; - async.waterfall([ - next => bucketPut(authInfo, testBucketPutRequest, log, next), - (corsHeaders, next) => - bucketPutACL(authInfo, testPutACLRequest, log, next), - (corsHeaders, next) => bucketGetACL(authInfo, - testGetACLRequest, log, next), - (result, corsHeaders, next) => parseString(result, next), - ], (err, result) => { - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Grantee[0] - .ID[0], canonicalID); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Permission[0], - 'FULL_CONTROL'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[1], undefined); - done(); - }); + async.waterfall( + [ + next => bucketPut(authInfo, testBucketPutRequest, log, next), + (corsHeaders, next) => bucketPutACL(authInfo, testPutACLRequest, log, next), + (corsHeaders, next) => bucketGetACL(authInfo, testGetACLRequest, log, next), + (result, corsHeaders, next) => parseString(result, next), + ], + (err, result) => { + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Grantee[0].ID[0], + canonicalID, + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Permission[0], + 'FULL_CONTROL', + ); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[1], undefined); + done(); + }, + ); }); it('should get a canned public-read-write ACL', done => { @@ -74,7 +76,7 @@ describe('bucketGetACL API', () => { bucketName, namespace, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-acl': 'public-read-write', }, url: '/?acl', @@ -82,35 +84,36 @@ describe('bucketGetACL API', () => { actionImplicitDenies: false, }; - async.waterfall([ - next => bucketPut(authInfo, testBucketPutRequest, log, next), - (corsHeaders, next) => - bucketPutACL(authInfo, testPutACLRequest, log, next), - (corsHeaders, next) => bucketGetACL(authInfo, testGetACLRequest, - log, next), - (result, corsHeaders, next) => parseString(result, next), - ], (err, result) => { - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Grantee[0] - .ID[0], canonicalID); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Permission[0], 'FULL_CONTROL'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[1].Grantee[0].URI[0], - constants.publicId); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[1] - .Permission[0], 'READ'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[2].Grantee[0].URI[0], - constants.publicId); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[2] - .Permission[0], 'WRITE'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[3], undefined); - done(); - }); + async.waterfall( + [ + next => bucketPut(authInfo, testBucketPutRequest, log, next), + (corsHeaders, next) => bucketPutACL(authInfo, testPutACLRequest, log, next), + (corsHeaders, next) => bucketGetACL(authInfo, testGetACLRequest, log, next), + (result, corsHeaders, next) => parseString(result, next), + ], + (err, result) => { + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Grantee[0].ID[0], + canonicalID, + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Permission[0], + 'FULL_CONTROL', + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[1].Grantee[0].URI[0], + constants.publicId, + ); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[1].Permission[0], 'READ'); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[2].Grantee[0].URI[0], + constants.publicId, + ); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[2].Permission[0], 'WRITE'); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[3], undefined); + done(); + }, + ); }); it('should get a canned public-read ACL', done => { @@ -118,7 +121,7 @@ describe('bucketGetACL API', () => { bucketName, namespace, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-acl': 'public-read', }, url: '/?acl', @@ -126,29 +129,31 @@ describe('bucketGetACL API', () => { actionImplicitDenies: false, }; - async.waterfall([ - next => bucketPut(authInfo, testBucketPutRequest, log, next), - (corsHeaders, next) => - bucketPutACL(authInfo, testPutACLRequest, log, next), - (corsHeaders, next) => bucketGetACL(authInfo, testGetACLRequest, - log, next), - (result, corsHeaders, next) => parseString(result, next), - ], (err, result) => { - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Grantee[0] - .ID[0], canonicalID); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Permission[0], 'FULL_CONTROL'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[1].Grantee[0].URI[0], - constants.publicId); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[1] - .Permission[0], 'READ'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[2], undefined); - done(); - }); + async.waterfall( + [ + next => bucketPut(authInfo, testBucketPutRequest, log, next), + (corsHeaders, next) => bucketPutACL(authInfo, testPutACLRequest, log, next), + (corsHeaders, next) => bucketGetACL(authInfo, testGetACLRequest, log, next), + (result, corsHeaders, next) => parseString(result, next), + ], + (err, result) => { + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Grantee[0].ID[0], + canonicalID, + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Permission[0], + 'FULL_CONTROL', + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[1].Grantee[0].URI[0], + constants.publicId, + ); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[1].Permission[0], 'READ'); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[2], undefined); + done(); + }, + ); }); it('should get a canned authenticated-read ACL', done => { @@ -156,7 +161,7 @@ describe('bucketGetACL API', () => { bucketName, namespace, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-acl': 'authenticated-read', }, url: '/?acl', @@ -164,30 +169,31 @@ describe('bucketGetACL API', () => { actionImplicitDenies: false, }; - async.waterfall([ - next => bucketPut(authInfo, testBucketPutRequest, log, next), - (corsHeaders, next) => - bucketPutACL(authInfo, testPutACLRequest, log, next), - (corsHeaders, next) => bucketGetACL(authInfo, testGetACLRequest, - log, next), - (result, corsHeaders, next) => parseString(result, next), - ], (err, result) => { - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Grantee[0] - .ID[0], canonicalID); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Permission[0], - 'FULL_CONTROL'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[1].Grantee[0] - .URI[0], constants.allAuthedUsersId); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[1] - .Permission[0], 'READ'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[2], undefined); - done(); - }); + async.waterfall( + [ + next => bucketPut(authInfo, testBucketPutRequest, log, next), + (corsHeaders, next) => bucketPutACL(authInfo, testPutACLRequest, log, next), + (corsHeaders, next) => bucketGetACL(authInfo, testGetACLRequest, log, next), + (result, corsHeaders, next) => parseString(result, next), + ], + (err, result) => { + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Grantee[0].ID[0], + canonicalID, + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Permission[0], + 'FULL_CONTROL', + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[1].Grantee[0].URI[0], + constants.allAuthedUsersId, + ); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[1].Permission[0], 'READ'); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[2], undefined); + done(); + }, + ); }); it('should get a canned log-delivery-write ACL', done => { @@ -195,7 +201,7 @@ describe('bucketGetACL API', () => { bucketName, namespace, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-acl': 'log-delivery-write', }, url: '/?acl', @@ -203,36 +209,36 @@ describe('bucketGetACL API', () => { actionImplicitDenies: false, }; - async.waterfall([ - next => bucketPut(authInfo, testBucketPutRequest, log, next), - (corsHeaders, next) => - bucketPutACL(authInfo, testPutACLRequest, log, next), - (corsHeaders, next) => bucketGetACL(authInfo, testGetACLRequest, - log, next), - (result, corsHeaders, next) => parseString(result, next), - ], (err, result) => { - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Grantee[0] - .ID[0], canonicalID); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Permission[0], - 'FULL_CONTROL'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[1].Grantee[0] - .URI[0], constants.logId); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[1] - .Permission[0], 'WRITE'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[2].Grantee[0] - .URI[0], constants.logId); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[2] - .Permission[0], 'READ_ACP'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[3], undefined); - done(); - }); + async.waterfall( + [ + next => bucketPut(authInfo, testBucketPutRequest, log, next), + (corsHeaders, next) => bucketPutACL(authInfo, testPutACLRequest, log, next), + (corsHeaders, next) => bucketGetACL(authInfo, testGetACLRequest, log, next), + (result, corsHeaders, next) => parseString(result, next), + ], + (err, result) => { + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Grantee[0].ID[0], + canonicalID, + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Permission[0], + 'FULL_CONTROL', + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[1].Grantee[0].URI[0], + constants.logId, + ); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[1].Permission[0], 'WRITE'); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[2].Grantee[0].URI[0], + constants.logId, + ); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[2].Permission[0], 'READ_ACP'); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[3], undefined); + done(); + }, + ); }); it('should get specifically set ACLs', done => { @@ -240,104 +246,96 @@ describe('bucketGetACL API', () => { bucketName, namespace, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-grant-full-control': - 'emailaddress="sampleaccount1@sampling.com"' + - ',emailaddress="sampleaccount2@sampling.com"', + 'emailaddress="sampleaccount1@sampling.com"' + ',emailaddress="sampleaccount2@sampling.com"', 'x-amz-grant-read': `uri=${constants.logId}`, 'x-amz-grant-write': `uri=${constants.publicId}`, - 'x-amz-grant-read-acp': - 'id=79a59df900b949e55d96a1e698fbacedfd6e09d98eac' + - 'f8f8d5218e7cd47ef2be', - 'x-amz-grant-write-acp': - 'id=79a59df900b949e55d96a1e698fbacedfd6e09d98eac' + - 'f8f8d5218e7cd47ef2bf', + 'x-amz-grant-read-acp': 'id=79a59df900b949e55d96a1e698fbacedfd6e09d98eac' + 'f8f8d5218e7cd47ef2be', + 'x-amz-grant-write-acp': 'id=79a59df900b949e55d96a1e698fbacedfd6e09d98eac' + 'f8f8d5218e7cd47ef2bf', }, url: '/?acl', query: { acl: '' }, actionImplicitDenies: false, }; - const canonicalIDforSample1 = - '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be'; - const canonicalIDforSample2 = - '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2bf'; + const canonicalIDforSample1 = '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be'; + const canonicalIDforSample2 = '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2bf'; - async.waterfall([ - next => bucketPut(authInfo, testBucketPutRequest, log, next), - (corsHeaders, next) => - bucketPutACL(authInfo, testPutACLRequest, log, next), - (corsHeaders, next) => bucketGetACL(authInfo, testGetACLRequest, - log, next), - (result, corsHeaders, next) => parseString(result, next), - ], (err, result) => { - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Grantee[0] - .ID[0], canonicalIDforSample1); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Grantee[0] - .DisplayName[0], 'sampleaccount1@sampling.com'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Permission[0], - 'FULL_CONTROL'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[1].Grantee[0] - .ID[0], canonicalIDforSample2); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[1].Grantee[0] - .DisplayName[0], 'sampleaccount2@sampling.com'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[1].Permission[0], - 'FULL_CONTROL'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[2].Grantee[0] - .ID[0], canonicalIDforSample2); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[2].Grantee[0] - .DisplayName[0], 'sampleaccount2@sampling.com'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[2].Permission[0], - 'WRITE_ACP'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[3].Grantee[0] - .ID[0], canonicalIDforSample1); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[3].Grantee[0] - .DisplayName[0], 'sampleaccount1@sampling.com'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[3].Permission[0], - 'READ_ACP'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[4].Grantee[0] - .URI[0], constants.publicId); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[4] - .Permission[0], 'WRITE'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[5].Grantee[0] - .URI[0], constants.logId); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[5] - .Permission[0], 'READ'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[6], undefined); - done(); - }); + async.waterfall( + [ + next => bucketPut(authInfo, testBucketPutRequest, log, next), + (corsHeaders, next) => bucketPutACL(authInfo, testPutACLRequest, log, next), + (corsHeaders, next) => bucketGetACL(authInfo, testGetACLRequest, log, next), + (result, corsHeaders, next) => parseString(result, next), + ], + (err, result) => { + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Grantee[0].ID[0], + canonicalIDforSample1, + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Grantee[0].DisplayName[0], + 'sampleaccount1@sampling.com', + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Permission[0], + 'FULL_CONTROL', + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[1].Grantee[0].ID[0], + canonicalIDforSample2, + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[1].Grantee[0].DisplayName[0], + 'sampleaccount2@sampling.com', + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[1].Permission[0], + 'FULL_CONTROL', + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[2].Grantee[0].ID[0], + canonicalIDforSample2, + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[2].Grantee[0].DisplayName[0], + 'sampleaccount2@sampling.com', + ); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[2].Permission[0], 'WRITE_ACP'); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[3].Grantee[0].ID[0], + canonicalIDforSample1, + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[3].Grantee[0].DisplayName[0], + 'sampleaccount1@sampling.com', + ); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[3].Permission[0], 'READ_ACP'); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[4].Grantee[0].URI[0], + constants.publicId, + ); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[4].Permission[0], 'WRITE'); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[5].Grantee[0].URI[0], + constants.logId, + ); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[5].Permission[0], 'READ'); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[6], undefined); + done(); + }, + ); }); - const grantsByURI = [ - constants.publicId, - constants.allAuthedUsersId, - constants.logId, - ]; + const grantsByURI = [constants.publicId, constants.allAuthedUsersId, constants.logId]; grantsByURI.forEach(uri => { - it('should get all ACLs when predefined group - ' + - `${uri} is used for multiple grants`, done => { + it('should get all ACLs when predefined group - ' + `${uri} is used for multiple grants`, done => { const testPutACLRequest = { bucketName, namespace, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-grant-full-control': `uri = ${uri}`, 'x-amz-grant-read': `uri = ${uri}`, 'x-amz-grant-write': `uri = ${uri}`, @@ -349,36 +347,35 @@ describe('bucketGetACL API', () => { actionImplicitDenies: false, }; - async.waterfall([ - next => bucketPut(authInfo, testBucketPutRequest, - log, next), (corsHeaders, next) => - bucketPutACL(authInfo, testPutACLRequest, log, next), - (corsHeaders, next) => bucketGetACL(authInfo, - testGetACLRequest, log, next), - (result, corsHeaders, next) => parseString(result, next), - ], (err, result) => { - assert.ifError(err); - const grants = - result.AccessControlPolicy.AccessControlList[0].Grant; - grants.forEach(grant => { - assert.strictEqual(grant.Permission.length, 1); - assert.strictEqual(grant.Grantee.length, 1); - assert.strictEqual(grant.Grantee[0].URI.length, 1); - assert.strictEqual(grant.Grantee[0].URI[0], `${uri}`); - }); - done(); - }); + async.waterfall( + [ + next => bucketPut(authInfo, testBucketPutRequest, log, next), + (corsHeaders, next) => bucketPutACL(authInfo, testPutACLRequest, log, next), + (corsHeaders, next) => bucketGetACL(authInfo, testGetACLRequest, log, next), + (result, corsHeaders, next) => parseString(result, next), + ], + (err, result) => { + assert.ifError(err); + const grants = result.AccessControlPolicy.AccessControlList[0].Grant; + grants.forEach(grant => { + assert.strictEqual(grant.Permission.length, 1); + assert.strictEqual(grant.Grantee.length, 1); + assert.strictEqual(grant.Grantee[0].URI.length, 1); + assert.strictEqual(grant.Grantee[0].URI[0], `${uri}`); + }); + done(); + }, + ); }); }); - it('should get all ACLs when predefined groups are used for ' + - 'more than one grant', done => { + it('should get all ACLs when predefined groups are used for ' + 'more than one grant', done => { const { allAuthedUsersId, publicId } = constants; const testPutACLRequest = { bucketName, namespace, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-grant-write': `uri = ${allAuthedUsersId} `, 'x-amz-grant-write-acp': `uri = ${allAuthedUsersId} `, 'x-amz-grant-read': `uri = ${publicId} `, @@ -389,33 +386,33 @@ describe('bucketGetACL API', () => { actionImplicitDenies: false, }; - async.waterfall([ - next => bucketPut(authInfo, testBucketPutRequest, log, next), - (corsHeaders, next) => - bucketPutACL(authInfo, testPutACLRequest, log, next), - (corsHeaders, next) => bucketGetACL(authInfo, testGetACLRequest, - log, next), - (result, corsHeaders, next) => parseString(result, next), - ], (err, result) => { - assert.ifError(err); - const grants = - result.AccessControlPolicy.AccessControlList[0].Grant; - grants.forEach(grant => { - const permissions = grant.Permission; - assert.strictEqual(permissions.length, 1); - const permission = permissions[0]; - assert.strictEqual(grant.Grantee.length, 1); - const grantees = grant.Grantee[0].URI; - assert.strictEqual(grantees.length, 1); - const grantee = grantees[0]; - if (['WRITE', 'WRITE_ACP'].includes(permission)) { - assert.strictEqual(grantee, constants.allAuthedUsersId); - } - if (['READ', 'READ_ACP'].includes(permission)) { - assert.strictEqual(grantee, constants.publicId); - } - }); - done(); - }); + async.waterfall( + [ + next => bucketPut(authInfo, testBucketPutRequest, log, next), + (corsHeaders, next) => bucketPutACL(authInfo, testPutACLRequest, log, next), + (corsHeaders, next) => bucketGetACL(authInfo, testGetACLRequest, log, next), + (result, corsHeaders, next) => parseString(result, next), + ], + (err, result) => { + assert.ifError(err); + const grants = result.AccessControlPolicy.AccessControlList[0].Grant; + grants.forEach(grant => { + const permissions = grant.Permission; + assert.strictEqual(permissions.length, 1); + const permission = permissions[0]; + assert.strictEqual(grant.Grantee.length, 1); + const grantees = grant.Grantee[0].URI; + assert.strictEqual(grantees.length, 1); + const grantee = grantees[0]; + if (['WRITE', 'WRITE_ACP'].includes(permission)) { + assert.strictEqual(grantee, constants.allAuthedUsersId); + } + if (['READ', 'READ_ACP'].includes(permission)) { + assert.strictEqual(grantee, constants.publicId); + } + }); + done(); + }, + ); }); }); diff --git a/tests/unit/api/bucketGetCors.js b/tests/unit/api/bucketGetCors.js index 01c8bf1839..675125f45c 100644 --- a/tests/unit/api/bucketGetCors.js +++ b/tests/unit/api/bucketGetCors.js @@ -4,10 +4,7 @@ const crypto = require('crypto'); const { bucketPut } = require('../../../lib/api/bucketPut'); const bucketPutCors = require('../../../lib/api/bucketPutCors'); const bucketGetCors = require('../../../lib/api/bucketGetCors'); -const { cleanup, - DummyRequestLogger, - makeAuthInfo } -= require('../helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../helpers'); const log = new DummyRequestLogger(); const authInfo = makeAuthInfo('accessKey1'); @@ -32,25 +29,24 @@ function _makeCorsRequest(xml) { if (xml) { request.post = xml; - request.headers['content-md5'] = crypto.createHash('md5') - .update(request.post, 'utf8').digest('base64'); + request.headers['content-md5'] = crypto.createHash('md5').update(request.post, 'utf8').digest('base64'); } return request; } const testGetCorsRequest = _makeCorsRequest(); function _comparePutGetXml(sampleXml, done) { - const fullXml = '' + - `${sampleXml}`; + const fullXml = + '' + + `${sampleXml}`; const testPutCorsRequest = _makeCorsRequest(fullXml); bucketPutCors(authInfo, testPutCorsRequest, log, err => { if (err) { process.stdout.write(`Err putting cors config ${err}`); return done(err); } - return bucketGetCors(authInfo, testGetCorsRequest, log, - (err, res) => { + return bucketGetCors(authInfo, testGetCorsRequest, log, (err, res) => { assert.strictEqual(err, null, `Unexpected err ${err}`); assert.strictEqual(res, fullXml); done(); @@ -65,8 +61,7 @@ describe('getBucketCors API', () => { }); afterEach(() => cleanup()); - it('should return same XML as uploaded for AllowedMethod and ' + - 'AllowedOrigin', done => { + it('should return same XML as uploaded for AllowedMethod and ' + 'AllowedOrigin', done => { const sampleXml = '' + 'PUT' + @@ -91,7 +86,7 @@ describe('getBucketCors API', () => { _comparePutGetXml(sampleXml, done); }); - it('should return same XML as uploaded for AllowedHeader\'s', done => { + it("should return same XML as uploaded for AllowedHeader's", done => { const sampleXml = '' + 'PUT' + @@ -103,7 +98,7 @@ describe('getBucketCors API', () => { _comparePutGetXml(sampleXml, done); }); - it('should return same XML as uploaded for ExposedHeader\'s', done => { + it("should return same XML as uploaded for ExposedHeader's", done => { const sampleXml = '' + 'PUT' + diff --git a/tests/unit/api/bucketGetLifecycle.js b/tests/unit/api/bucketGetLifecycle.js index 95025d74a7..d80d336b72 100644 --- a/tests/unit/api/bucketGetLifecycle.js +++ b/tests/unit/api/bucketGetLifecycle.js @@ -3,12 +3,8 @@ const assert = require('assert'); const { bucketPut } = require('../../../lib/api/bucketPut'); const bucketGetLifecycle = require('../../../lib/api/bucketGetLifecycle'); const bucketPutLifecycle = require('../../../lib/api/bucketPutLifecycle'); -const { cleanup, - DummyRequestLogger, - makeAuthInfo } - = require('../helpers'); -const { getLifecycleRequest, getLifecycleXml } = - require('../utils/lifecycleHelpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../helpers'); +const { getLifecycleRequest, getLifecycleXml } = require('../utils/lifecycleHelpers'); const log = new DummyRequestLogger(); const authInfo = makeAuthInfo('accessKey1'); @@ -25,8 +21,7 @@ describe('getBucketLifecycle API', () => { beforeEach(done => bucketPut(authInfo, testBucketPutRequest, log, done)); afterEach(() => cleanup()); - it('should return NoSuchLifecycleConfiguration error if ' + - 'bucket has no lifecycle', done => { + it('should return NoSuchLifecycleConfiguration error if ' + 'bucket has no lifecycle', done => { const lifecycleRequest = getLifecycleRequest(bucketName); bucketGetLifecycle(authInfo, lifecycleRequest, log, err => { assert.strictEqual(err.is.NoSuchLifecycleConfiguration, true); @@ -36,8 +31,7 @@ describe('getBucketLifecycle API', () => { describe('after bucket lifecycle has been put', () => { beforeEach(done => { - const putRequest = - getLifecycleRequest(bucketName, getLifecycleXml()); + const putRequest = getLifecycleRequest(bucketName, getLifecycleXml()); bucketPutLifecycle(authInfo, putRequest, log, err => { assert.equal(err, null); done(); @@ -48,8 +42,7 @@ describe('getBucketLifecycle API', () => { const getRequest = getLifecycleRequest(bucketName); bucketGetLifecycle(authInfo, getRequest, log, (err, res) => { assert.equal(err, null); - const expectedXML = '' + - `${getLifecycleXml()}`; + const expectedXML = '' + `${getLifecycleXml()}`; assert.deepStrictEqual(expectedXML, res); done(); }); diff --git a/tests/unit/api/bucketGetLocation.js b/tests/unit/api/bucketGetLocation.js index 27f0815263..46ef4e2dc5 100644 --- a/tests/unit/api/bucketGetLocation.js +++ b/tests/unit/api/bucketGetLocation.js @@ -2,16 +2,10 @@ const assert = require('assert'); const { bucketPut } = require('../../../lib/api/bucketPut'); const bucketGetLocation = require('../../../lib/api/bucketGetLocation'); -const { cleanup, - DummyRequestLogger, - makeAuthInfo } -= require('../helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../helpers'); const { config } = require('../../../lib/Config'); -const { - LOCATION_NAME_DMF, - LOCATION_NAME_CRR, -} = require('../../constants'); +const { LOCATION_NAME_DMF, LOCATION_NAME_CRR } = require('../../constants'); const log = new DummyRequestLogger(); const authInfo = makeAuthInfo('accessKey1'); @@ -37,11 +31,13 @@ const testGetLocationRequest = { const locationConstraints = config.locationConstraints; function getBucketRequestObject(location) { - const post = location ? '' + - '' + - `${location}` + - '' : undefined; + const post = location + ? '' + + '' + + `${location}` + + '' + : undefined; return Object.assign({ post }, testBucketPutRequest); } @@ -64,13 +60,11 @@ describe('getBucketLocation API', () => { }); afterEach(() => cleanup()); it(`should return ${location} LocationConstraint xml`, done => { - bucketGetLocation(authInfo, testGetLocationRequest, log, - (err, res) => { - assert.strictEqual(err, null, - `Unexpected ${err} getting location constraint`); - const xml = ` - ` + - `${location}`; + bucketGetLocation(authInfo, testGetLocationRequest, log, (err, res) => { + assert.strictEqual(err, null, `Unexpected ${err} getting location constraint`); + const xml = + ` + ` + `${location}`; assert.deepStrictEqual(res, xml); return done(); }); @@ -86,13 +80,11 @@ describe('getBucketLocation API', () => { }); afterEach(() => cleanup()); it('should return empty string LocationConstraint xml', done => { - bucketGetLocation(authInfo, testGetLocationRequest, log, - (err, res) => { - assert.strictEqual(err, null, - `Unexpected ${err} getting location constraint`); - const xml = ` - ` + - ''; + bucketGetLocation(authInfo, testGetLocationRequest, log, (err, res) => { + assert.strictEqual(err, null, `Unexpected ${err} getting location constraint`); + const xml = + ` + ` + ''; assert.deepStrictEqual(res, xml); return done(); }); diff --git a/tests/unit/api/bucketGetLogging.js b/tests/unit/api/bucketGetLogging.js index d6bb09a950..acce545928 100644 --- a/tests/unit/api/bucketGetLogging.js +++ b/tests/unit/api/bucketGetLogging.js @@ -61,13 +61,15 @@ function createGetLoggingRequest(bucketName, headers = {}) { } function createValidLoggingXML(targetBucket, targetPrefix = 'logs/') { - return '' + + return ( + '' + '' + '' + `${targetBucket}` + `${targetPrefix}` + '' + - ''; + '' + ); } describe('bucketGetLogging API', () => { @@ -90,7 +92,8 @@ describe('bucketGetLogging API', () => { assert.ifError(err); assert(xml); // Should return empty BucketLoggingStatus - const expectedXML = '\n' + + const expectedXML = + '\n' + ''; assert.strictEqual(xml, expectedXML); done(); @@ -163,7 +166,8 @@ describe('bucketGetLogging API', () => { assert.ifError(err); // Disable logging - const disableXML = '' + + const disableXML = + '' + ''; const disableRequest = createLoggingRequest(bucketName, disableXML); diff --git a/tests/unit/api/bucketGetNotification.js b/tests/unit/api/bucketGetNotification.js index 5091b45ef9..f48606d411 100644 --- a/tests/unit/api/bucketGetNotification.js +++ b/tests/unit/api/bucketGetNotification.js @@ -3,10 +3,7 @@ const assert = require('assert'); const { bucketPut } = require('../../../lib/api/bucketPut'); const bucketGetNotification = require('../../../lib/api/bucketGetNotification'); const bucketPutNotification = require('../../../lib/api/bucketPutNotification'); -const { cleanup, - DummyRequestLogger, - makeAuthInfo } - = require('../helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../helpers'); const log = new DummyRequestLogger(); const authInfo = makeAuthInfo('accessKey1'); @@ -40,7 +37,8 @@ function getNotificationXml() { const filterName = 'Prefix'; const filterValue = 'logs/'; - return '' + + return ( + '' + '' + `${id}` + `${queueArn}` + @@ -51,10 +49,10 @@ function getNotificationXml() { `${filterValue}` + '' + '' + - ''; + '' + ); } - describe('getBucketNotification API', () => { before(cleanup); beforeEach(done => bucketPut(authInfo, testBucketPutRequest, log, done)); @@ -70,8 +68,7 @@ describe('getBucketNotification API', () => { describe('after bucket notification has been put', () => { beforeEach(done => { - const putRequest = - getNotificationRequest(bucketName, getNotificationXml()); + const putRequest = getNotificationRequest(bucketName, getNotificationXml()); bucketPutNotification(authInfo, putRequest, log, err => { assert.ifError(err); done(); @@ -82,8 +79,7 @@ describe('getBucketNotification API', () => { const getRequest = getNotificationRequest(bucketName); bucketGetNotification(authInfo, getRequest, log, (err, res) => { assert.ifError(err); - const expectedXML = '' + - `${getNotificationXml()}`; + const expectedXML = '' + `${getNotificationXml()}`; assert.deepStrictEqual(expectedXML, res); done(); }); diff --git a/tests/unit/api/bucketGetObjectLock.js b/tests/unit/api/bucketGetObjectLock.js index 39c55574e7..e5553da33b 100644 --- a/tests/unit/api/bucketGetObjectLock.js +++ b/tests/unit/api/bucketGetObjectLock.js @@ -20,7 +20,7 @@ const bucketPutReq = { const testBucketPutReqWithObjLock = { bucketName, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-bucket-object-lock-enabled': 'True', }, url: '/', @@ -31,7 +31,7 @@ function getObjectLockConfigRequest(bucketName, xml) { const request = { bucketName, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-bucket-object-lock-enabled': 'true', }, url: '/?object-lock', @@ -59,10 +59,7 @@ function getObjectLockXml(mode, type, time) { // object lock is enabled and object lock configuration is set if (arguments.length === 3) { - xmlStr += xml.ruleOpen + - retentionMode + - retentionTime + - xml.ruleClose; + xmlStr += xml.ruleOpen + retentionMode + retentionTime + xml.ruleClose; } xmlStr += xml.objLockConfigClose; return xmlStr; @@ -72,14 +69,16 @@ describe('bucketGetObjectLock API', () => { before(done => bucketPut(authInfo, bucketPutReq, log, done)); after(cleanup); - it('should return ObjectLockConfigurationNotFoundError error if ' + - 'object lock is not enabled on the bucket', done => { - const objectLockRequest = getObjectLockConfigRequest(bucketName); - bucketGetObjectLock(authInfo, objectLockRequest, log, err => { - assert.strictEqual(err.is.ObjectLockConfigurationNotFoundError, true); - done(); - }); - }); + it( + 'should return ObjectLockConfigurationNotFoundError error if ' + 'object lock is not enabled on the bucket', + done => { + const objectLockRequest = getObjectLockConfigRequest(bucketName); + bucketGetObjectLock(authInfo, objectLockRequest, log, err => { + assert.strictEqual(err.is.ObjectLockConfigurationNotFoundError, true); + done(); + }); + }, + ); }); describe('bucketGetObjectLock API', () => { @@ -87,8 +86,7 @@ describe('bucketGetObjectLock API', () => { beforeEach(done => bucketPut(authInfo, testBucketPutReqWithObjLock, log, done)); afterEach(cleanup); - it('should return config without \'rule\' if object lock configuration ' + - 'not set on the bucket', done => { + it("should return config without 'rule' if object lock configuration " + 'not set on the bucket', done => { const objectLockRequest = getObjectLockConfigRequest(bucketName); bucketGetObjectLock(authInfo, objectLockRequest, log, (err, res) => { assert.ifError(err); diff --git a/tests/unit/api/bucketGetPolicy.js b/tests/unit/api/bucketGetPolicy.js index 504f702c92..b51e378390 100644 --- a/tests/unit/api/bucketGetPolicy.js +++ b/tests/unit/api/bucketGetPolicy.js @@ -3,10 +3,7 @@ const assert = require('assert'); const { bucketPut } = require('../../../lib/api/bucketPut'); const bucketGetPolicy = require('../../../lib/api/bucketGetPolicy'); const bucketPutPolicy = require('../../../lib/api/bucketPutPolicy'); -const { cleanup, - DummyRequestLogger, - makeAuthInfo } - = require('../helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../helpers'); const log = new DummyRequestLogger(); const authInfo = makeAuthInfo('accessKey1'); @@ -43,8 +40,7 @@ describe('getBucketPolicy API', () => { beforeEach(done => bucketPut(authInfo, testBasicRequest, log, done)); afterEach(() => cleanup()); - it('should return NoSuchBucketPolicy error if ' + - 'bucket has no policy', done => { + it('should return NoSuchBucketPolicy error if ' + 'bucket has no policy', done => { bucketGetPolicy(authInfo, testBasicRequest, log, err => { assert.strictEqual(err.is.NoSuchBucketPolicy, true); done(); diff --git a/tests/unit/api/bucketGetRateLimit.js b/tests/unit/api/bucketGetRateLimit.js index 2f65447795..e43625b62f 100644 --- a/tests/unit/api/bucketGetRateLimit.js +++ b/tests/unit/api/bucketGetRateLimit.js @@ -85,8 +85,7 @@ describe('bucketGetRateLimit API', () => { }); }); - it('should return NoSuchRateLimitConfig error if bucket exists but ' + - 'rate limit config is not set', done => { + it('should return NoSuchRateLimitConfig error if bucket exists but ' + 'rate limit config is not set', done => { bucketPut(regularAuthInfo, bucketPutReq, log, err => { assert.ifError(err); const rateLimitRequest = getRateLimitConfigRequest(bucketName); diff --git a/tests/unit/api/bucketGetReplication.js b/tests/unit/api/bucketGetReplication.js index 268c902025..be29801cf2 100644 --- a/tests/unit/api/bucketGetReplication.js +++ b/tests/unit/api/bucketGetReplication.js @@ -2,8 +2,7 @@ const assert = require('assert'); const { parseString } = require('xml2js'); const { DummyRequestLogger } = require('../helpers'); -const { getReplicationConfigurationXML } = - require('../../../lib/api/apiUtils/bucket/getReplicationConfiguration'); +const { getReplicationConfigurationXML } = require('../../../lib/api/apiUtils/bucket/getReplicationConfiguration'); // Compare the values from the parsedXML with the original configuration values. function checkXML(parsedXML, config) { @@ -58,29 +57,25 @@ describe("'getReplicationConfigurationXML' function", () => { it('should return XML from the bucket replication configuration', done => getAndCheckXML(getReplicationConfig(), done)); - it('should not return XML with StorageClass tag if `storageClass` ' + - 'property is omitted', done => { + it('should not return XML with StorageClass tag if `storageClass` ' + 'property is omitted', done => { const config = getReplicationConfig(); delete config.rules[0].storageClass; return getAndCheckXML(config, done); }); - it("should return XML with StorageClass tag set to 'Disabled' if " + - '`enabled` property is false', done => { + it("should return XML with StorageClass tag set to 'Disabled' if " + '`enabled` property is false', done => { const config = getReplicationConfig(); config.rules[0].enabled = false; return getAndCheckXML(config, done); }); - it('should return XML with a self-closing Prefix tag if `prefix` ' + - "property is ''", done => { + it('should return XML with a self-closing Prefix tag if `prefix` ' + "property is ''", done => { const config = getReplicationConfig(); config.rules[0].prefix = ''; return getAndCheckXML(config, done); }); - it('should return XML from the bucket replication configuration with ' + - 'multiple rules', done => { + it('should return XML from the bucket replication configuration with ' + 'multiple rules', done => { const config = getReplicationConfig(); config.rules.push({ id: 'test-id-2', diff --git a/tests/unit/api/bucketGetTagging.js b/tests/unit/api/bucketGetTagging.js index a1c5f502db..686d92d243 100644 --- a/tests/unit/api/bucketGetTagging.js +++ b/tests/unit/api/bucketGetTagging.js @@ -1,11 +1,7 @@ const assert = require('assert'); const { bucketPut } = require('../../../lib/api/bucketPut'); -const { cleanup, - DummyRequestLogger, - makeAuthInfo, - TaggingConfigTester, -} = require('../helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo, TaggingConfigTester } = require('../helpers'); const bucketPutTagging = require('../../../lib/api/bucketPutTagging'); const bucketGetTagging = require('../../../lib/api/bucketGetTagging'); const log = new DummyRequestLogger(); @@ -29,14 +25,11 @@ describe('getBucketTagging API', () => { it('should return tags resource', done => { const taggingUtil = new TaggingConfigTester(); - const testBucketPutTaggingRequest = taggingUtil - .createBucketTaggingRequest('PUT', bucketName); + const testBucketPutTaggingRequest = taggingUtil.createBucketTaggingRequest('PUT', bucketName); bucketPutTagging(authInfo, testBucketPutTaggingRequest, log, err => { assert.strictEqual(err, undefined); - const testBucketGetTaggingRequest = taggingUtil - .createBucketTaggingRequest('GET', bucketName); - return bucketGetTagging(authInfo, testBucketGetTaggingRequest, log, - (err, xml) => { + const testBucketGetTaggingRequest = taggingUtil.createBucketTaggingRequest('GET', bucketName); + return bucketGetTagging(authInfo, testBucketGetTaggingRequest, log, (err, xml) => { if (err) { process.stdout.write(`Err getting object tagging ${err}`); return done(err); @@ -49,19 +42,15 @@ describe('getBucketTagging API', () => { it('should return access denied if the authorization check fails', done => { const taggingUtil = new TaggingConfigTester(); - const testBucketPutTaggingRequest = taggingUtil - .createBucketTaggingRequest('PUT', bucketName); + const testBucketPutTaggingRequest = taggingUtil.createBucketTaggingRequest('PUT', bucketName); bucketPutTagging(authInfo, testBucketPutTaggingRequest, log, err => { assert.strictEqual(err, undefined); - const testBucketGetTaggingRequest = taggingUtil - .createBucketTaggingRequest('GET', bucketName, true); + const testBucketGetTaggingRequest = taggingUtil.createBucketTaggingRequest('GET', bucketName, true); const badAuthInfo = makeAuthInfo('accessKey2'); - return bucketGetTagging(badAuthInfo, testBucketGetTaggingRequest, log, - err => { + return bucketGetTagging(badAuthInfo, testBucketGetTaggingRequest, log, err => { assert.strictEqual(err.AccessDenied, true); return done(); }); }); }); }); - diff --git a/tests/unit/api/bucketGetWebsite.js b/tests/unit/api/bucketGetWebsite.js index 75caf14129..eb3c9fc999 100644 --- a/tests/unit/api/bucketGetWebsite.js +++ b/tests/unit/api/bucketGetWebsite.js @@ -3,10 +3,7 @@ const assert = require('assert'); const { bucketPut } = require('../../../lib/api/bucketPut'); const bucketPutWebsite = require('../../../lib/api/bucketPutWebsite'); const bucketGetWebsite = require('../../../lib/api/bucketGetWebsite'); -const { cleanup, - DummyRequestLogger, - makeAuthInfo } -= require('../helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../helpers'); const log = new DummyRequestLogger(); const authInfo = makeAuthInfo('accessKey1'); @@ -37,18 +34,18 @@ function _makeWebsiteRequest(xml) { const testGetWebsiteRequest = _makeWebsiteRequest(); function _comparePutGetXml(sampleXml, done) { - const fullXml = '' + - `${sampleXml}`; + const fullXml = + '' + + `${sampleXml}`; const testPutWebsiteRequest = _makeWebsiteRequest(fullXml); bucketPutWebsite(authInfo, testPutWebsiteRequest, log, err => { if (err) { process.stdout.write(`Err putting website config ${err}`); return done(err); } - return bucketGetWebsite(authInfo, testGetWebsiteRequest, log, - (err, res) => { + return bucketGetWebsite(authInfo, testGetWebsiteRequest, log, (err, res) => { assert.strictEqual(err, null, `Unexpected err ${err}`); assert.strictEqual(res, fullXml); done(); @@ -64,8 +61,7 @@ describe('getBucketWebsite API', () => { afterEach(() => cleanup()); it('should return same IndexDocument XML as uploaded', done => { - const sampleXml = - 'index.html'; + const sampleXml = 'index.html'; _comparePutGetXml(sampleXml, done); }); it('should return same ErrorDocument XML as uploaded', done => { diff --git a/tests/unit/api/bucketHead.js b/tests/unit/api/bucketHead.js index 3c006a549c..b2fa311b30 100644 --- a/tests/unit/api/bucketHead.js +++ b/tests/unit/api/bucketHead.js @@ -37,8 +37,7 @@ describe('bucketHead API', () => { }); }); - it('should return no error if bucket exists and user is authorized', - done => { + it('should return no error if bucket exists and user is authorized', done => { bucketPut(authInfo, testRequest, log, () => { bucketHead(authInfo, testRequest, log, err => { assert.strictEqual(err, null); diff --git a/tests/unit/api/bucketPolicyAuth.js b/tests/unit/api/bucketPolicyAuth.js index 618f7b8908..67f4a3297b 100644 --- a/tests/unit/api/bucketPolicyAuth.js +++ b/tests/unit/api/bucketPolicyAuth.js @@ -2,8 +2,11 @@ const assert = require('assert'); const { BucketInfo, BucketPolicy } = require('arsenal').models; const AuthInfo = require('arsenal').auth.AuthInfo; const constants = require('../../../constants'); -const { isBucketAuthorized, isObjAuthorized, validatePolicyResource } - = require('../../../lib/api/apiUtils/authorization/permissionChecks'); +const { + isBucketAuthorized, + isObjAuthorized, + validatePolicyResource, +} = require('../../../lib/api/apiUtils/authorization/permissionChecks'); const { DummyRequestLogger, makeAuthInfo } = require('../helpers'); const DummyRequest = require('../DummyRequest'); @@ -20,8 +23,12 @@ const altAcctCanonicalId = altAcctAuthInfo.getCanonicalID(); const accountId = authInfo.getShortid(); const altAcctId = altAcctAuthInfo.getShortid(); const creationDate = new Date().toJSON(); -const bucket = new BucketInfo('policyBucketAuthTester', bucketOwnerCanonicalId, - authInfo.getAccountDisplayName(), creationDate); +const bucket = new BucketInfo( + 'policyBucketAuthTester', + bucketOwnerCanonicalId, + authInfo.getAccountDisplayName(), + creationDate, +); const object = { 'owner-id': objectOwnerCanonicalId }; const bucAction = 'bucketHead'; const objAction = 'objectPut'; @@ -90,8 +97,7 @@ const authTests = [ expected: true, }, { - name: 'should allow access if account id principal is contained in ' + - 'user arn of non-', + name: 'should allow access if account id principal is contained in ' + 'user arn of non-', bucketId: objectOwnerCanonicalId, bucketAuthInfo: user1AuthInfo, objectId: objectOwnerCanonicalId, @@ -103,8 +109,7 @@ const authTests = [ expected: true, }, { - name: 'should allow access if account id principal is contained in ' + - 'account arn of non-', + name: 'should allow access if account id principal is contained in ' + 'account arn of non-', bucketId: altAcctCanonicalId, bucketAuthInfo: altAcctAuthInfo, objectId: altAcctCanonicalId, @@ -116,8 +121,7 @@ const authTests = [ expected: true, }, { - name: 'should allow access if account arn principal is contained in ' + - 'user arn of non-', + name: 'should allow access if account arn principal is contained in ' + 'user arn of non-', bucketId: objectOwnerCanonicalId, bucketAuthInfo: user1AuthInfo, objectId: objectOwnerCanonicalId, @@ -129,8 +133,7 @@ const authTests = [ expected: true, }, { - name: 'should allow access even if user arn principal doesn\'t match ' + - 'user arn of user in account of ', + name: "should allow access even if user arn principal doesn't match " + 'user arn of user in account of ', bucketId: objectOwnerCanonicalId, bucketAuthInfo: user1AuthInfo, objectId: objectOwnerCanonicalId, @@ -142,8 +145,7 @@ const authTests = [ expected: true, }, { - name: 'should deny access if account arn principal doesn\'t match ' + - 'user arn of non-', + name: "should deny access if account arn principal doesn't match " + 'user arn of non-', bucketId: altAcctCanonicalId, bucketAuthInfo: altAcctUserAuthInfo, objectId: altAcctCanonicalId, @@ -155,8 +157,7 @@ const authTests = [ expected: false, }, { - name: 'should deny access if user arn principal doesn\'t match ' + - 'user arn of non-', + name: "should deny access if user arn principal doesn't match " + 'user arn of non-', bucketId: altAcctCanonicalId, bucketAuthInfo: altAcctUserAuthInfo, objectId: altAcctCanonicalId, @@ -168,7 +169,7 @@ const authTests = [ expected: false, }, { - name: 'should deny access if principal doesn\'t match non-', + name: "should deny access if principal doesn't match non-", bucketId: altAcctCanonicalId, bucketAuthInfo: altAcctAuthInfo, objectId: altAcctCanonicalId, @@ -180,8 +181,7 @@ const authTests = [ expected: false, }, { - name: 'should allow access if principal and action match policy for ' + - 'non-', + name: 'should allow access if principal and action match policy for ' + 'non-', bucketId: altAcctCanonicalId, bucketAuthInfo: altAcctAuthInfo, objectId: altAcctCanonicalId, @@ -193,8 +193,7 @@ const authTests = [ expected: true, }, { - name: 'should deny access if principal matches but action does not ' + - 'match policy for non-', + name: 'should deny access if principal matches but action does not ' + 'match policy for non-', bucketId: altAcctCanonicalId, bucketAuthInfo: altAcctAuthInfo, objectId: altAcctCanonicalId, @@ -253,8 +252,7 @@ const resourceTests = [ expected: false, }, { - name: 'false if policy resource is array and any elements do not ' + - 'match bucket arn', + name: 'false if policy resource is array and any elements do not ' + 'match bucket arn', rValue: [`arn:aws:s3:::${bucketName}`, 'arn:aws:s3:::nomatch'], expected: false, }, @@ -263,50 +261,41 @@ const resourceTests = [ describe('bucket policy authorization', () => { describe('isBucketAuthorized with no policy set', () => { it('should allow access to bucket owner', done => { - const allowed = isBucketAuthorized(bucket, 'bucketPut', - bucketOwnerCanonicalId, null, log); + const allowed = isBucketAuthorized(bucket, 'bucketPut', bucketOwnerCanonicalId, null, log); assert.equal(allowed, true); done(); }); - it('should deny access to non-bucket owner', - done => { - const allowed = isBucketAuthorized(bucket, 'bucketPut', - altAcctCanonicalId, null, log); - assert.equal(allowed, false); - done(); - }); + it('should deny access to non-bucket owner', done => { + const allowed = isBucketAuthorized(bucket, 'bucketPut', altAcctCanonicalId, null, log); + assert.equal(allowed, false); + done(); + }); }); describe('isBucketAuthorized with bucket policy set', () => { beforeEach(function beFn() { - this.currentTest.basePolicy = new BucketPolicy(JSON.stringify( - basePolicyObj)).getBucketPolicy(); + this.currentTest.basePolicy = new BucketPolicy(JSON.stringify(basePolicyObj)).getBucketPolicy(); bucket.setBucketPolicy(this.currentTest.basePolicy); }); - it('should allow access to non-bucket owner if principal is set to "*"', - done => { - const allowed = isBucketAuthorized(bucket, bucAction, - altAcctCanonicalId, null, log); - assert.equal(allowed, true); - done(); - }); + it('should allow access to non-bucket owner if principal is set to "*"', done => { + const allowed = isBucketAuthorized(bucket, bucAction, altAcctCanonicalId, null, log); + assert.equal(allowed, true); + done(); + }); - it('should allow access to public user if principal is set to "*"', - done => { - const allowed = isBucketAuthorized(bucket, bucAction, - constants.publicId, publicUserAuthInfo, log); - assert.equal(allowed, true); - done(); - }); + it('should allow access to public user if principal is set to "*"', done => { + const allowed = isBucketAuthorized(bucket, bucAction, constants.publicId, publicUserAuthInfo, log); + assert.equal(allowed, true); + done(); + }); it('should deny access to public user if principal is not set to "*"', function itFn(done) { const newPolicy = this.test.basePolicy; newPolicy.Statement[0].Principal = { AWS: authInfo.getArn() }; bucket.setBucketPolicy(newPolicy); - const allowed = isBucketAuthorized(bucket, bucAction, - constants.publicId, publicUserAuthInfo, log); + const allowed = isBucketAuthorized(bucket, bucAction, constants.publicId, publicUserAuthInfo, log); assert.equal(allowed, false); done(); }); @@ -316,37 +305,35 @@ describe('bucket policy authorization', () => { const newPolicy = this.test.basePolicy; newPolicy.Statement[0][t.keyToChange] = t.bucketValue; bucket.setBucketPolicy(newPolicy); - const allowed = isBucketAuthorized(bucket, bucAction, - t.bucketId, t.bucketAuthInfo, log); + const allowed = isBucketAuthorized(bucket, bucAction, t.bucketId, t.bucketAuthInfo, log); assert.equal(allowed, t.expected); done(); }); }); - it('should deny access to non-bucket owner if two statements apply ' + - 'to principal but one denies access', function itFn(done) { - const newPolicy = this.test.basePolicy; - newPolicy.Statement[1] = { - Effect: 'Deny', - Principal: { CanonicalUser: [altAcctCanonicalId] }, - Resource: `arn:aws:s3:::${bucket.getName()}`, - Action: 's3:*', - }; - bucket.setBucketPolicy(newPolicy); - const allowed = isBucketAuthorized(bucket, bucAction, - altAcctCanonicalId, null, log); + it( + 'should deny access to non-bucket owner if two statements apply ' + 'to principal but one denies access', + function itFn(done) { + const newPolicy = this.test.basePolicy; + newPolicy.Statement[1] = { + Effect: 'Deny', + Principal: { CanonicalUser: [altAcctCanonicalId] }, + Resource: `arn:aws:s3:::${bucket.getName()}`, + Action: 's3:*', + }; + bucket.setBucketPolicy(newPolicy); + const allowed = isBucketAuthorized(bucket, bucAction, altAcctCanonicalId, null, log); + assert.equal(allowed, false); + done(); + }, + ); + + it('should deny access to non-bucket owner with an unsupported action type', done => { + const allowed = isBucketAuthorized(bucket, 'unsupportedAction', altAcctCanonicalId, null, log); assert.equal(allowed, false); done(); }); - it('should deny access to non-bucket owner with an unsupported action type', - done => { - const allowed = isBucketAuthorized(bucket, 'unsupportedAction', - altAcctCanonicalId, null, log); - assert.equal(allowed, false); - done(); - }); - it('should bypass bucket policy when request.bypassUserBucketPolicies is true', function () { // Create a request with the bypassUserBucketPolicies flag initially not set const request = { @@ -365,14 +352,12 @@ describe('bucket policy authorization', () => { bucket.setBucketPolicy(newPolicy); // Check that the policy denies access, as expected - assert.ok(!isBucketAuthorized(bucket, bucAction, - bucketOwnerCanonicalId, user1AuthInfo, log, request)); + assert.ok(!isBucketAuthorized(bucket, bucAction, bucketOwnerCanonicalId, user1AuthInfo, log, request)); // But with bypassUserBucketPolicies set to true, it should still be authorized // based on ACL permissions (which we mock to return true) request.bypassUserBucketPolicies = true; - assert.ok(isBucketAuthorized(bucket, bucAction, - bucketOwnerCanonicalId, user1AuthInfo, log, request)); + assert.ok(isBucketAuthorized(bucket, bucAction, bucketOwnerCanonicalId, user1AuthInfo, log, request)); }); }); @@ -382,79 +367,74 @@ describe('bucket policy authorization', () => { }); it('should allow access to object owner', done => { - const allowed = isObjAuthorized(bucket, object, objAction, - objectOwnerCanonicalId, null, log); + const allowed = isObjAuthorized(bucket, object, objAction, objectOwnerCanonicalId, null, log); assert.equal(allowed, true); done(); }); - it('should deny access to non-object owner', - done => { - const allowed = isObjAuthorized(bucket, object, objAction, - altAcctCanonicalId, null, log); - assert.equal(allowed, false); - done(); - }); + it('should deny access to non-object owner', done => { + const allowed = isObjAuthorized(bucket, object, objAction, altAcctCanonicalId, null, log); + assert.equal(allowed, false); + done(); + }); }); describe('isObjAuthorized with bucket policy set', () => { beforeEach(function beFn() { const newPolicyObj = basePolicyObj; - newPolicyObj.Statement.Resource = - `arn:aws:s3:::${bucket.getName()}/*`; - this.currentTest.basePolicy = new BucketPolicy(JSON.stringify( - newPolicyObj)).getBucketPolicy(); + newPolicyObj.Statement.Resource = `arn:aws:s3:::${bucket.getName()}/*`; + this.currentTest.basePolicy = new BucketPolicy(JSON.stringify(newPolicyObj)).getBucketPolicy(); bucket.setBucketPolicy(this.currentTest.basePolicy); }); - it('should allow access to non-object owner if principal is set to "*"', - done => { - const allowed = isObjAuthorized(bucket, object, objAction, - altAcctCanonicalId, null, log); - assert.equal(allowed, true); - done(); - }); + it('should allow access to non-object owner if principal is set to "*"', done => { + const allowed = isObjAuthorized(bucket, object, objAction, altAcctCanonicalId, null, log); + assert.equal(allowed, true); + done(); + }); - it('should allow access to public user if principal is set to "*"', - done => { - const allowed = isObjAuthorized(bucket, object, objAction, - constants.publicId, publicUserAuthInfo, log); - assert.equal(allowed, true); - done(); - }); + it('should allow access to public user if principal is set to "*"', done => { + const allowed = isObjAuthorized(bucket, object, objAction, constants.publicId, publicUserAuthInfo, log); + assert.equal(allowed, true); + done(); + }); authTests.forEach(t => { it(`${t.name}object owner`, function itFn(done) { const newPolicy = this.test.basePolicy; newPolicy.Statement[0][t.keyToChange] = t.objectValue; bucket.setBucketPolicy(newPolicy); - const allowed = isObjAuthorized(bucket, object, objAction, - t.objectId, t.objectAuthInfo, log, null, t.impDenies); + const allowed = isObjAuthorized( + bucket, + object, + objAction, + t.objectId, + t.objectAuthInfo, + log, + null, + t.impDenies, + ); assert.equal(allowed, t.expected); done(); }); }); - it('should allow access to non-object owner for objectHead action with s3:GetObject permission', - function itFn(done) { - const newPolicy = this.test.basePolicy; - newPolicy.Statement[0].Action = ['s3:GetObject']; - bucket.setBucketPolicy(newPolicy); - const allowed = isObjAuthorized(bucket, object, 'objectHead', - altAcctCanonicalId, altAcctAuthInfo, log); - assert.equal(allowed, true); - done(); - }); - it('should deny access to non-object owner for objectHead action without s3:GetObject permission', - function itFn(done) { - const newPolicy = this.test.basePolicy; - newPolicy.Statement[0].Action = ['s3:PutObject']; - bucket.setBucketPolicy(newPolicy); - const allowed = isObjAuthorized(bucket, object, 'objectHead', - altAcctCanonicalId, altAcctAuthInfo, log); - assert.equal(allowed, false); - done(); - }); + it('should allow access to non-object owner for objectHead action with s3:GetObject permission', function itFn(done) { + const newPolicy = this.test.basePolicy; + newPolicy.Statement[0].Action = ['s3:GetObject']; + bucket.setBucketPolicy(newPolicy); + const allowed = isObjAuthorized(bucket, object, 'objectHead', altAcctCanonicalId, altAcctAuthInfo, log); + assert.equal(allowed, true); + done(); + }); + it('should deny access to non-object owner for objectHead action without s3:GetObject permission', function itFn(done) { + const newPolicy = this.test.basePolicy; + newPolicy.Statement[0].Action = ['s3:PutObject']; + bucket.setBucketPolicy(newPolicy); + const allowed = isObjAuthorized(bucket, object, 'objectHead', altAcctCanonicalId, altAcctAuthInfo, log); + assert.equal(allowed, false); + done(); + }); it('should bypass bucket policy when request.bypassUserBucketPolicies is true', function () { // Create a request with the bypassUserBucketPolicies flag initially not set @@ -474,40 +454,41 @@ describe('bucket policy authorization', () => { bucket.setBucketPolicy(newPolicy); // Check that the policy denies access, as expected - assert.ok(!isObjAuthorized(bucket, object, 'objectGet', - bucketOwnerCanonicalId, user1AuthInfo, log, request)); + assert.ok( + !isObjAuthorized(bucket, object, 'objectGet', bucketOwnerCanonicalId, user1AuthInfo, log, request), + ); // But with bypassUserBucketPolicies set to true, it should still be authorized // based on ACL permissions (which we mock to return true) request.bypassUserBucketPolicies = true; - assert.ok(isObjAuthorized(bucket, object, 'objectGet', - bucketOwnerCanonicalId, user1AuthInfo, log, request)); + assert.ok( + isObjAuthorized(bucket, object, 'objectGet', bucketOwnerCanonicalId, user1AuthInfo, log, request), + ); }); - it('should deny access to non-object owner if two statements apply ' + - 'to principal but one denies access', function itFn(done) { - const newPolicy = this.test.basePolicy; - newPolicy.Statement[1] = { - Effect: 'Deny', - Principal: { CanonicalUser: [altAcctCanonicalId] }, - Resource: `arn:aws:s3:::${bucket.getName()}/*`, - Action: 's3:*', - }; - bucket.setBucketPolicy(newPolicy); - const allowed = isObjAuthorized(bucket, object, objAction, - altAcctCanonicalId, null, log); + it( + 'should deny access to non-object owner if two statements apply ' + 'to principal but one denies access', + function itFn(done) { + const newPolicy = this.test.basePolicy; + newPolicy.Statement[1] = { + Effect: 'Deny', + Principal: { CanonicalUser: [altAcctCanonicalId] }, + Resource: `arn:aws:s3:::${bucket.getName()}/*`, + Action: 's3:*', + }; + bucket.setBucketPolicy(newPolicy); + const allowed = isObjAuthorized(bucket, object, objAction, altAcctCanonicalId, null, log); + assert.equal(allowed, false); + done(); + }, + ); + + it('should deny access to non-object owner with an unsupported action type', done => { + const allowed = isObjAuthorized(bucket, object, 'unsupportedAction', altAcctCanonicalId, null, log); assert.equal(allowed, false); done(); }); - it('should deny access to non-object owner with an unsupported action type', - done => { - const allowed = isObjAuthorized(bucket, object, 'unsupportedAction', - altAcctCanonicalId, null, log); - assert.equal(allowed, false); - done(); - }); - it('should allow access when implicitDeny true with Allow bucket policy', function itFn() { const requestTypes = ['objectPut', 'objectDelete']; const impDenies = { @@ -523,8 +504,16 @@ describe('bucket policy authorization', () => { try { const results = requestTypes.map(type => { - const allowed = isObjAuthorized(bucket, object, type, - altAcctCanonicalId, altAcctAuthInfo, log, null, impDenies); + const allowed = isObjAuthorized( + bucket, + object, + type, + altAcctCanonicalId, + altAcctAuthInfo, + log, + null, + impDenies, + ); return allowed; }); assert.deepStrictEqual(results, [true, true]); @@ -550,8 +539,16 @@ describe('bucket policy authorization', () => { bucket.setBucketPolicy(newPolicy); const results = requestTypes.map(type => { - const allowed = isObjAuthorized(bucket, object, type, - altAcctCanonicalId, altAcctAuthInfo, log, null, impDenies); + const allowed = isObjAuthorized( + bucket, + object, + type, + altAcctCanonicalId, + altAcctAuthInfo, + log, + null, + impDenies, + ); return allowed; }); assert.deepStrictEqual(results, [false, false]); @@ -564,14 +561,12 @@ describe('bucket policy authorization', () => { const newPolicy = basePolicyObj; newPolicy.Statement.Resource = t.rValue; newPolicy.Statement = [newPolicy.Statement]; - assert.equal( - validatePolicyResource(bucketName, newPolicy), t.expected); + assert.equal(validatePolicyResource(bucketName, newPolicy), t.expected); done(); }); }); - it('should return false if any statement resource does not match ' + - 'bucket arn', done => { + it('should return false if any statement resource does not match ' + 'bucket arn', done => { const newPolicy = basePolicyObj; newPolicy.Statement = [newPolicy.Statement]; newPolicy.Statement[1] = basePolicyObj.Statement; @@ -591,23 +586,15 @@ describe('bucket policy authorization', () => { CanonicalUser: [altAcctCanonicalId], }, Action: 's3:*', - Resource: [ - `arn:aws:s3:::${bucket.getName()}`, - `arn:aws:s3:::${bucket.getName()}/*`, - ], + Resource: [`arn:aws:s3:::${bucket.getName()}`, `arn:aws:s3:::${bucket.getName()}/*`], }, { Effect: 'Deny', Principal: { CanonicalUser: [altAcctCanonicalId], }, - Action: [ - 's3:PutObjectRetention', - ], - Resource: [ - `arn:aws:s3:::${bucket.getName()}`, - `arn:aws:s3:::${bucket.getName()}/*`, - ], + Action: ['s3:PutObjectRetention'], + Resource: [`arn:aws:s3:::${bucket.getName()}`, `arn:aws:s3:::${bucket.getName()}/*`], Condition: { NumericGreaterThan: { 's3:object-lock-remaining-retention-days': 10, @@ -626,10 +613,7 @@ describe('bucket policy authorization', () => { CanonicalUser: [altAcctCanonicalId], }, Action: 's3:*', - Resource: [ - `arn:aws:s3:::${bucket.getName()}`, - `arn:aws:s3:::${bucket.getName()}/*`, - ], + Resource: [`arn:aws:s3:::${bucket.getName()}`, `arn:aws:s3:::${bucket.getName()}/*`], Condition: { IpAddress: { 'aws:SourceIp': '123.123.123.123', @@ -692,11 +676,17 @@ describe('bucket policy authorization', () => { requestParams[t.requestConditionKey] = t.conditionValue; const request = new DummyRequest(requestParams); - const results = isObjAuthorized(bucket, object, t.requestType, - altAcctCanonicalId, altAcctAuthInfo, log, request); + const results = isObjAuthorized( + bucket, + object, + t.requestType, + altAcctCanonicalId, + altAcctAuthInfo, + log, + request, + ); assert.strictEqual(results, t.expectedVerdict); - } - ); + }); }); }); }); diff --git a/tests/unit/api/bucketPutACL.js b/tests/unit/api/bucketPutACL.js index 7c08b760ca..44953126c7 100644 --- a/tests/unit/api/bucketPutACL.js +++ b/tests/unit/api/bucketPutACL.js @@ -17,21 +17,15 @@ const testBucketPutRequest = { headers: { host: `${bucketName}.s3.amazonaws.com` }, url: '/', }; -const canonicalIDforSample1 = - '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be'; -const canonicalIDforSample2 = - '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2bf'; +const canonicalIDforSample1 = '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be'; +const canonicalIDforSample2 = '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2bf'; const invalidIds = { 'too short': 'id="invalid_id"', - 'too long': - 'id=79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2aaa', - 'only numbers': - 'id=0000000000000000000000000000000000000000000000000000000000000000', - 'only letters': - 'id=abcdefabcdefabcdefabcdefabcdefacbdefabcdefabcdefabcdefabcdefabcd', - 'non-hex letters': - 'id=79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2ZZ', + 'too long': 'id=79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2aaa', + 'only numbers': 'id=0000000000000000000000000000000000000000000000000000000000000000', + 'only letters': 'id=abcdefabcdefabcdefabcdefabcdefacbdefabcdefabcdefabcdefabcdefabcd', + 'non-hex letters': 'id=79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2ZZ', }; describe('putBucketACL API', () => { @@ -65,7 +59,7 @@ describe('putBucketACL API', () => { bucketName, namespace, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-acl': 'not-a-valid-option', }, url: '/?acl', @@ -84,7 +78,7 @@ describe('putBucketACL API', () => { bucketName, namespace, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-acl': 'public-read-write', }, url: '/?acl', @@ -100,13 +94,12 @@ describe('putBucketACL API', () => { }); }); - it('should set a canned public-read ACL followed by ' - + 'a canned authenticated-read ACL', done => { + it('should set a canned public-read ACL followed by ' + 'a canned authenticated-read ACL', done => { const testACLRequest = { bucketName, namespace, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-acl': 'public-read', }, url: '/?acl', @@ -117,7 +110,7 @@ describe('putBucketACL API', () => { bucketName, namespace, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-acl': 'authenticated-read', }, url: '/?acl', @@ -131,8 +124,7 @@ describe('putBucketACL API', () => { bucketPutACL(authInfo, testACLRequest2, log, err => { assert.strictEqual(err, undefined); metadata.getBucket(bucketName, log, (err, md) => { - assert.strictEqual(md.getAcl().Canned, - 'authenticated-read'); + assert.strictEqual(md.getAcl().Canned, 'authenticated-read'); done(); }); }); @@ -140,13 +132,12 @@ describe('putBucketACL API', () => { }); }); - it('should set a canned private ACL ' + - 'followed by a log-delivery-write ACL', done => { + it('should set a canned private ACL ' + 'followed by a log-delivery-write ACL', done => { const testACLRequest = { bucketName, namespace, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-acl': 'private', }, url: '/?acl', @@ -157,7 +148,7 @@ describe('putBucketACL API', () => { bucketName, namespace, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-acl': 'log-delivery-write', }, url: '/?acl', @@ -172,8 +163,7 @@ describe('putBucketACL API', () => { bucketPutACL(authInfo, testACLRequest2, log, err => { assert.strictEqual(err, undefined); metadata.getBucket(bucketName, log, (err, md) => { - assert.strictEqual(md.getAcl().Canned, - 'log-delivery-write'); + assert.strictEqual(md.getAcl().Canned, 'log-delivery-write'); done(); }); }); @@ -186,18 +176,13 @@ describe('putBucketACL API', () => { bucketName, namespace, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-grant-full-control': - 'emailaddress="sampleaccount1@sampling.com"' + - ',emailaddress="sampleaccount2@sampling.com"', + 'emailaddress="sampleaccount1@sampling.com"' + ',emailaddress="sampleaccount2@sampling.com"', 'x-amz-grant-read': `uri=${constants.logId}`, 'x-amz-grant-write': `uri=${constants.publicId}`, - 'x-amz-grant-read-acp': - 'id=79a59df900b949e55d96a1e698fbacedfd6e09d98eac' + - 'f8f8d5218e7cd47ef2be', - 'x-amz-grant-write-acp': - 'id=79a59df900b949e55d96a1e698fbacedfd6e09d98eac' + - 'f8f8d5218e7cd47ef2bf', + 'x-amz-grant-read-acp': 'id=79a59df900b949e55d96a1e698fbacedfd6e09d98eac' + 'f8f8d5218e7cd47ef2be', + 'x-amz-grant-write-acp': 'id=79a59df900b949e55d96a1e698fbacedfd6e09d98eac' + 'f8f8d5218e7cd47ef2bf', }, url: '/?acl', query: { acl: '' }, @@ -207,95 +192,77 @@ describe('putBucketACL API', () => { assert.strictEqual(err, undefined); metadata.getBucket(bucketName, log, (err, md) => { assert.strictEqual(md.getAcl().WRITE[0], constants.publicId); - assert(md.getAcl().FULL_CONTROL - .indexOf(canonicalIDforSample1) > -1); - assert(md.getAcl().FULL_CONTROL - .indexOf(canonicalIDforSample2) > -1); - assert(md.getAcl().READ_ACP - .indexOf(canonicalIDforSample1) > -1); - assert(md.getAcl().WRITE_ACP - .indexOf(canonicalIDforSample2) > -1); + assert(md.getAcl().FULL_CONTROL.indexOf(canonicalIDforSample1) > -1); + assert(md.getAcl().FULL_CONTROL.indexOf(canonicalIDforSample2) > -1); + assert(md.getAcl().READ_ACP.indexOf(canonicalIDforSample1) > -1); + assert(md.getAcl().WRITE_ACP.indexOf(canonicalIDforSample2) > -1); done(); }); }); }); - it('should set all ACLs sharing the same email in request headers', - done => { - const testACLRequest = { - bucketName, - namespace, - headers: { - 'host': `${bucketName}.s3.amazonaws.com`, - 'x-amz-grant-full-control': - 'emailaddress="sampleaccount1@sampling.com"' + - ',emailaddress="sampleaccount2@sampling.com"', - 'x-amz-grant-read': - 'emailaddress="sampleaccount1@sampling.com"', - 'x-amz-grant-write': - 'emailaddress="sampleaccount1@sampling.com"', - 'x-amz-grant-read-acp': - 'id=79a59df900b949e55d96a1e698fbacedfd6e09d98eac' + - 'f8f8d5218e7cd47ef2be', - 'x-amz-grant-write-acp': - 'id=79a59df900b949e55d96a1e698fbacedfd6e09d98eac' + - 'f8f8d5218e7cd47ef2bf', - }, - url: '/?acl', - query: { acl: '' }, - actionImplicitDenies: false, - }; - bucketPutACL(authInfo, testACLRequest, log, err => { - assert.strictEqual(err, undefined); - metadata.getBucket(bucketName, log, (err, md) => { - assert(md.getAcl().WRITE.indexOf(canonicalIDforSample1) - > -1); - assert(md.getAcl().READ.indexOf(canonicalIDforSample1) - > -1); - assert(md.getAcl().FULL_CONTROL - .indexOf(canonicalIDforSample1) > -1); - assert(md.getAcl().FULL_CONTROL - .indexOf(canonicalIDforSample2) > -1); - assert(md.getAcl().READ_ACP - .indexOf(canonicalIDforSample1) > -1); - assert(md.getAcl().WRITE_ACP - .indexOf(canonicalIDforSample2) > -1); - done(); - }); + it('should set all ACLs sharing the same email in request headers', done => { + const testACLRequest = { + bucketName, + namespace, + headers: { + host: `${bucketName}.s3.amazonaws.com`, + 'x-amz-grant-full-control': + 'emailaddress="sampleaccount1@sampling.com"' + ',emailaddress="sampleaccount2@sampling.com"', + 'x-amz-grant-read': 'emailaddress="sampleaccount1@sampling.com"', + 'x-amz-grant-write': 'emailaddress="sampleaccount1@sampling.com"', + 'x-amz-grant-read-acp': 'id=79a59df900b949e55d96a1e698fbacedfd6e09d98eac' + 'f8f8d5218e7cd47ef2be', + 'x-amz-grant-write-acp': 'id=79a59df900b949e55d96a1e698fbacedfd6e09d98eac' + 'f8f8d5218e7cd47ef2bf', + }, + url: '/?acl', + query: { acl: '' }, + actionImplicitDenies: false, + }; + bucketPutACL(authInfo, testACLRequest, log, err => { + assert.strictEqual(err, undefined); + metadata.getBucket(bucketName, log, (err, md) => { + assert(md.getAcl().WRITE.indexOf(canonicalIDforSample1) > -1); + assert(md.getAcl().READ.indexOf(canonicalIDforSample1) > -1); + assert(md.getAcl().FULL_CONTROL.indexOf(canonicalIDforSample1) > -1); + assert(md.getAcl().FULL_CONTROL.indexOf(canonicalIDforSample2) > -1); + assert(md.getAcl().READ_ACP.indexOf(canonicalIDforSample1) > -1); + assert(md.getAcl().WRITE_ACP.indexOf(canonicalIDforSample2) > -1); + done(); }); }); + }); Object.keys(invalidIds).forEach(idType => { - it('should return an error if grantee canonical ID provided in ACL ' + - `request invalid because ${idType}`, done => { - const testACLRequest = { - bucketName, - namespace, - headers: { - 'host': `${bucketName}.s3.amazonaws.com`, - 'x-amz-grant-full-control': invalidIds[idType], - }, - url: '/?acl', - query: { acl: '' }, - actionImplicitDenies: false, - }; - return bucketPutACL(authInfo, testACLRequest, log, err => { - assert.strictEqual(err.is.InvalidArgument, true); - done(); - }); - }); + it( + 'should return an error if grantee canonical ID provided in ACL ' + `request invalid because ${idType}`, + done => { + const testACLRequest = { + bucketName, + namespace, + headers: { + host: `${bucketName}.s3.amazonaws.com`, + 'x-amz-grant-full-control': invalidIds[idType], + }, + url: '/?acl', + query: { acl: '' }, + actionImplicitDenies: false, + }; + return bucketPutACL(authInfo, testACLRequest, log, err => { + assert.strictEqual(err.is.InvalidArgument, true); + done(); + }); + }, + ); }); - it('should return an error if invalid email ' + - 'provided in ACL header request', done => { + it('should return an error if invalid email ' + 'provided in ACL header request', done => { const testACLRequest = { bucketName, namespace, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-grant-full-control': - 'emailaddress="sampleaccount1@sampling.com"' + - ',emailaddress="nonexistentEmail@sampling.com"', + 'emailaddress="sampleaccount1@sampling.com"' + ',emailaddress="nonexistentEmail@sampling.com"', }, url: '/?acl', query: { acl: '' }, @@ -313,49 +280,50 @@ describe('putBucketACL API', () => { bucketName, namespace, headers: { host: `${bucketName}.s3.amazonaws.com` }, - post: '' + - '' + - '79a59df900b949e55d96a1e698fbaced' + - 'fd6e09d98eacf8f8d5218e7cd47ef2be' + - 'OwnerDisplayName' + - '' + - '' + - '' + - '' + - '79a59df900b949e55d96a1e698fbaced' + - 'fd6e09d98eacf8f8d5218e7cd47ef2be' + - 'OwnerDisplayName' + - '' + - 'FULL_CONTROL' + - '' + - '' + - '' + - `${constants.publicId}` + - '' + - 'READ' + - '' + - '' + - '' + - `${constants.logId}` + - '' + - 'WRITE' + - '' + - '' + - '' + - 'sampleaccount1@sampling.com' + - '' + - '' + - 'WRITE_ACP' + - '' + - '' + - '' + - '79a59df900b949e55d96a1e698fbacedfd' + - '6e09d98eacf8f8d5218e7cd47ef2bf' + - '' + - 'READ_ACP' + - '' + - '' + + post: + '' + + '' + + '79a59df900b949e55d96a1e698fbaced' + + 'fd6e09d98eacf8f8d5218e7cd47ef2be' + + 'OwnerDisplayName' + + '' + + '' + + '' + + '' + + '79a59df900b949e55d96a1e698fbaced' + + 'fd6e09d98eacf8f8d5218e7cd47ef2be' + + 'OwnerDisplayName' + + '' + + 'FULL_CONTROL' + + '' + + '' + + '' + + `${constants.publicId}` + + '' + + 'READ' + + '' + + '' + + '' + + `${constants.logId}` + + '' + + 'WRITE' + + '' + + '' + + '' + + 'sampleaccount1@sampling.com' + + '' + + '' + + 'WRITE_ACP' + + '' + + '' + + '' + + '79a59df900b949e55d96a1e698fbacedfd' + + '6e09d98eacf8f8d5218e7cd47ef2bf' + + '' + + 'READ_ACP' + + '' + + '' + '', url: '/?acl', query: { acl: '' }, @@ -366,14 +334,11 @@ describe('putBucketACL API', () => { assert.strictEqual(err, undefined); metadata.getBucket(bucketName, log, (err, md) => { assert.strictEqual(md.getAcl().Canned, ''); - assert.strictEqual(md.getAcl().FULL_CONTROL[0], - canonicalIDforSample1); + assert.strictEqual(md.getAcl().FULL_CONTROL[0], canonicalIDforSample1); assert.strictEqual(md.getAcl().READ[0], constants.publicId); assert.strictEqual(md.getAcl().WRITE[0], constants.logId); - assert.strictEqual(md.getAcl().WRITE_ACP[0], - canonicalIDforSample1); - assert.strictEqual(md.getAcl().READ_ACP[0], - canonicalIDforSample2); + assert.strictEqual(md.getAcl().WRITE_ACP[0], canonicalIDforSample1); + assert.strictEqual(md.getAcl().READ_ACP[0], canonicalIDforSample2); done(); }); }); @@ -384,14 +349,15 @@ describe('putBucketACL API', () => { bucketName, namespace, headers: { host: `${bucketName}.s3.amazonaws.com` }, - post: '' + - '' + - '79a59df900b949e55d96a1e698fbaced' + - 'fd6e09d98eacf8f8d5218e7cd47ef2be' + - 'OwnerDisplayName' + - '' + - '' + + post: + '' + + '' + + '79a59df900b949e55d96a1e698fbaced' + + 'fd6e09d98eacf8f8d5218e7cd47ef2be' + + 'OwnerDisplayName' + + '' + + '' + '', url: '/?acl', query: { acl: '' }, @@ -412,19 +378,19 @@ describe('putBucketACL API', () => { }); }); - it('should not be able to set ACLs without AccessControlList section', - done => { + it('should not be able to set ACLs without AccessControlList section', done => { const testACLRequest = { bucketName, namespace, headers: { host: `${bucketName}.s3.amazonaws.com` }, - post: '' + - '' + - '79a59df900b949e55d96a1e698fbaced' + - 'fd6e09d98eacf8f8d5218e7cd47ef2be' + - 'OwnerDisplayName' + - '' + + post: + '' + + '' + + '79a59df900b949e55d96a1e698fbaced' + + 'fd6e09d98eacf8f8d5218e7cd47ef2be' + + 'OwnerDisplayName' + + '' + '', url: '/?acl', query: { acl: '' }, @@ -442,33 +408,34 @@ describe('putBucketACL API', () => { bucketName, namespace, headers: { host: `${bucketName}.s3.amazonaws.com` }, - post: '' + - '' + - '79a59df900b949e55d96a1e698fbaced' + - 'fd6e09d98eacf8f8d5218e7cd47ef2be' + - 'OwnerDisplayName' + - '' + - '' + - '' + - '' + - '79a59df900b949e55d96a1e698fbaced' + - 'fd6e09d98eacf8f8d5218e7cd47ef2be' + - 'OwnerDisplayName' + - '' + - 'FULL_CONTROL' + - '' + - '' + - '' + - '' + - '' + - '79a59df900b949e55d96a1e698fbaced' + - 'fd6e09d98eacf8f8d5218e7cd47ef2be' + - 'OwnerDisplayName' + - '' + - 'READ' + - '' + - '' + + post: + '' + + '' + + '79a59df900b949e55d96a1e698fbaced' + + 'fd6e09d98eacf8f8d5218e7cd47ef2be' + + 'OwnerDisplayName' + + '' + + '' + + '' + + '' + + '79a59df900b949e55d96a1e698fbaced' + + 'fd6e09d98eacf8f8d5218e7cd47ef2be' + + 'OwnerDisplayName' + + '' + + 'FULL_CONTROL' + + '' + + '' + + '' + + '' + + '' + + '79a59df900b949e55d96a1e698fbaced' + + 'fd6e09d98eacf8f8d5218e7cd47ef2be' + + 'OwnerDisplayName' + + '' + + 'READ' + + '' + + '' + '', url: '/?acl', query: { acl: '' }, @@ -481,27 +448,27 @@ describe('putBucketACL API', () => { }); }); - it('should return an error if invalid grantee user ID ' + - 'provided in ACL request body', done => { + it('should return an error if invalid grantee user ID ' + 'provided in ACL request body', done => { const testACLRequest = { bucketName, namespace, headers: { host: `${bucketName}.s3.amazonaws.com` }, - post: '' + - '' + - '79a59df900b949e55d96a1e698fbaced' + - 'fd6e09d98eacf8f8d5218e7cd47ef2be' + - 'OwnerDisplayName' + - '' + - '' + - '' + - '' + - 'invalid_id' + - '' + - 'READ_ACP' + - '' + - '' + + post: + '' + + '' + + '79a59df900b949e55d96a1e698fbaced' + + 'fd6e09d98eacf8f8d5218e7cd47ef2be' + + 'OwnerDisplayName' + + '' + + '' + + '' + + '' + + 'invalid_id' + + '' + + 'READ_ACP' + + '' + + '' + '', url: '/?acl', query: { acl: '' }, @@ -514,27 +481,27 @@ describe('putBucketACL API', () => { }); }); - it('should return an error if invalid email ' + - 'address provided in ACLs set out in request body', done => { + it('should return an error if invalid email ' + 'address provided in ACLs set out in request body', done => { const testACLRequest = { bucketName, namespace, headers: { host: `${bucketName}.s3.amazonaws.com` }, - post: '' + - '' + - '79a59df900b949e55d96a1e698fbaced' + - 'fd6e09d98eacf8f8d5218e7cd47ef2be' + - 'OwnerDisplayName' + - '' + - '' + - '' + - '' + - 'xyz@amazon.com' + - '' + - 'WRITE_ACP' + - '' + - '' + + post: + '' + + '' + + '79a59df900b949e55d96a1e698fbaced' + + 'fd6e09d98eacf8f8d5218e7cd47ef2be' + + 'OwnerDisplayName' + + '' + + '' + + '' + + '' + + 'xyz@amazon.com' + + '' + + 'WRITE_ACP' + + '' + + '' + '', url: '/?acl', query: { acl: '' }, @@ -546,31 +513,31 @@ describe('putBucketACL API', () => { }); }); - it('should return an error if xml provided does not match s3 ' - + 'scheme for setting ACLs', done => { + it('should return an error if xml provided does not match s3 ' + 'scheme for setting ACLs', done => { const testACLRequest = { bucketName, namespace, headers: { host: `${bucketName}.s3.amazonaws.com` }, /** XML below uses the term "PowerGrant" instead of - * "Grant" which is part of the s3 xml scheme for ACLs - * so an error should be returned - */ - post: '' + - '' + - '79a59df900b949e55d96a1e698fbaced' + - 'fd6e09d98eacf8f8d5218e7cd47ef2be' + - 'OwnerDisplayName' + - '' + - '' + - '' + - '' + - 'xyz@amazon.com' + - '' + - 'WRITE_ACP' + - '' + - '' + + * "Grant" which is part of the s3 xml scheme for ACLs + * so an error should be returned + */ + post: + '' + + '' + + '79a59df900b949e55d96a1e698fbaced' + + 'fd6e09d98eacf8f8d5218e7cd47ef2be' + + 'OwnerDisplayName' + + '' + + '' + + '' + + '' + + 'xyz@amazon.com' + + '' + + 'WRITE_ACP' + + '' + + '' + '', url: '/?acl', query: { acl: '' }, @@ -583,51 +550,54 @@ describe('putBucketACL API', () => { }); }); - - it('should return an error if xml provided does not match s3 ' - + 'scheme for setting ACLs using multiple Grant section', done => { - const testACLRequest = { - bucketName, - namespace, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - /** XML below uses the term "PowerGrant" instead of - * "Grant" which is part of the s3 xml scheme for ACLs - * so an error should be returned - */ - post: ' { + const testACLRequest = { + bucketName, + namespace, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + /** XML below uses the term "PowerGrant" instead of + * "Grant" which is part of the s3 xml scheme for ACLs + * so an error should be returned + */ + post: + '' + - '' + + '' + '79a59df900b949e55d96a1e698fbaced' + 'fd6e09d98eacf8f8d5218e7cd47ef2be' + 'OwnerDisplayName' + - '' + - '' + + '' + + '' + '' + - '' + - '79a59df900b949e55d96a1e698fbaced' + - 'fd6e09d98eacf8f8d5218e7cd47ef2be' + - 'OwnerDisplayName' + - '' + - 'FULL_CONTROL' + + '' + + '79a59df900b949e55d96a1e698fbaced' + + 'fd6e09d98eacf8f8d5218e7cd47ef2be' + + 'OwnerDisplayName' + + '' + + 'FULL_CONTROL' + '' + '' + - '' + - 'xyz@amazon.com' + - '' + - 'WRITE_ACP' + + '' + + 'xyz@amazon.com' + + '' + + 'WRITE_ACP' + '' + - '' + - '', - url: '/?acl', - query: { acl: '' }, - actionImplicitDenies: false, - }; + '' + + '', + url: '/?acl', + query: { acl: '' }, + actionImplicitDenies: false, + }; - bucketPutACL(authInfo, testACLRequest, log, err => { - assert.strictEqual(err.is.MalformedACLError, true); - done(); - }); - }); + bucketPutACL(authInfo, testACLRequest, log, err => { + assert.strictEqual(err.is.MalformedACLError, true); + done(); + }); + }, + ); it('should return an error if malformed xml provided', done => { const testACLRequest = { @@ -639,20 +609,20 @@ describe('putBucketACL API', () => { post: { '' + - '' + + '' + '79a59df900b949e55d96a1e698fbaced' + 'fd6e09d98eacf8f8d5218e7cd47ef2be' + 'OwnerDisplayName' + - '' + - '' + + '' + + '' + '' + - '' + - 'xyz@amazon.com' + - '' + - 'WRITE_ACP' + + '' + + 'xyz@amazon.com' + + '' + + 'WRITE_ACP' + '' + - '' + - '', + '' + + '', }, url: '/?acl', query: { acl: '' }, @@ -665,29 +635,29 @@ describe('putBucketACL API', () => { }); }); - it('should return an error if invalid group ' + - 'uri provided in ACLs set out in request body', done => { + it('should return an error if invalid group ' + 'uri provided in ACLs set out in request body', done => { const testACLRequest = { bucketName, namespace, headers: { host: `${bucketName}.s3.amazonaws.com` }, // URI in grant below is not valid group URI for s3 - post: '' + - '' + - '79a59df900b949e55d96a1e698fbaced' + - 'fd6e09d98eacf8f8d5218e7cd47ef2be' + - 'OwnerDisplayName' + - '' + - '' + - '' + - '' + - 'http://acs.amazonaws.com/groups/' + - 'global/NOTAVALIDGROUP' + - '' + - 'READ' + - '' + - '' + + post: + '' + + '' + + '79a59df900b949e55d96a1e698fbaced' + + 'fd6e09d98eacf8f8d5218e7cd47ef2be' + + 'OwnerDisplayName' + + '' + + '' + + '' + + '' + + 'http://acs.amazonaws.com/groups/' + + 'global/NOTAVALIDGROUP' + + '' + + 'READ' + + '' + + '' + '', url: '/?acl', query: { acl: '' }, @@ -700,16 +670,13 @@ describe('putBucketACL API', () => { }); }); - it('should return an error if invalid group uri' + - 'provided in ACL header request', done => { + it('should return an error if invalid group uri' + 'provided in ACL header request', done => { const testACLRequest = { bucketName, namespace, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, - 'x-amz-grant-full-control': - 'uri="http://acs.amazonaws.com/groups/' + - 'global/NOTAVALIDGROUP"', + host: `${bucketName}.s3.amazonaws.com`, + 'x-amz-grant-full-control': 'uri="http://acs.amazonaws.com/groups/' + 'global/NOTAVALIDGROUP"', }, url: '/?acl', query: { acl: '' }, diff --git a/tests/unit/api/bucketPutCors.js b/tests/unit/api/bucketPutCors.js index fc9e7bdcf2..e8c381d810 100644 --- a/tests/unit/api/bucketPutCors.js +++ b/tests/unit/api/bucketPutCors.js @@ -3,13 +3,8 @@ const { errors } = require('arsenal'); const { bucketPut } = require('../../../lib/api/bucketPut'); const bucketPutCors = require('../../../lib/api/bucketPutCors'); -const { _validator, parseCorsXml } - = require('../../../lib/api/apiUtils/bucket/bucketCors'); -const { cleanup, - DummyRequestLogger, - makeAuthInfo, - CorsConfigTester } - = require('../helpers'); +const { _validator, parseCorsXml } = require('../../../lib/api/apiUtils/bucket/bucketCors'); +const { cleanup, DummyRequestLogger, makeAuthInfo, CorsConfigTester } = require('../helpers'); const metadata = require('../../../lib/metadata/wrapper'); const log = new DummyRequestLogger(); @@ -31,13 +26,14 @@ function _testPutBucketCors(authInfo, request, log, errCode, cb) { } function _generateSampleXml(value) { - const xml = '' + - '' + - 'PUT' + - 'www.example.com' + - `${value}` + - '' + - ''; + const xml = + '' + + '' + + 'PUT' + + 'www.example.com' + + `${value}` + + '' + + ''; return xml; } @@ -49,10 +45,9 @@ describe('putBucketCORS API', () => { }); afterEach(() => cleanup()); - it('should update a bucket\'s metadata with cors resource', done => { + it("should update a bucket's metadata with cors resource", done => { const corsUtil = new CorsConfigTester(); - const testBucketPutCorsRequest = corsUtil - .createBucketCorsRequest('PUT', bucketName); + const testBucketPutCorsRequest = corsUtil.createBucketCorsRequest('PUT', bucketName); bucketPutCors(authInfo, testBucketPutCorsRequest, log, err => { if (err) { process.stdout.write(`Err putting bucket cors ${err}`); @@ -73,18 +68,14 @@ describe('putBucketCORS API', () => { it('should return MalformedXML if body greater than 64KB', done => { const corsUtil = new CorsConfigTester(); const body = Buffer.alloc(65537); // 64 * 1024 = 65536 bytes - const testBucketPutCorsRequest = corsUtil - .createBucketCorsRequest('PUT', bucketName, body); - _testPutBucketCors(authInfo, testBucketPutCorsRequest, - log, 'MalformedXML', done); + const testBucketPutCorsRequest = corsUtil.createBucketCorsRequest('PUT', bucketName, body); + _testPutBucketCors(authInfo, testBucketPutCorsRequest, log, 'MalformedXML', done); }); it('should return InvalidRequest if more than one MaxAgeSeconds', done => { const corsUtil = new CorsConfigTester({ maxAgeSeconds: [60, 6000] }); - const testBucketPutCorsRequest = corsUtil - .createBucketCorsRequest('PUT', bucketName); - _testPutBucketCors(authInfo, testBucketPutCorsRequest, - log, 'MalformedXML', done); + const testBucketPutCorsRequest = corsUtil.createBucketCorsRequest('PUT', bucketName); + _testPutBucketCors(authInfo, testBucketPutCorsRequest, log, 'MalformedXML', done); }); }); @@ -95,8 +86,7 @@ describe('PUT bucket cors :: helper validation functions ', () => { const expectedResults = [true, true, false]; for (let i = 0; i < testStrings.length; i++) { - const result = _validator - .validateNumberWildcards(testStrings[i]); + const result = _validator.validateNumberWildcards(testStrings[i]); assert.strictEqual(result, expectedResults[i]); } done(); @@ -117,8 +107,7 @@ describe('PUT bucket cors :: helper validation functions ', () => { it('should return MalformedXML if more than one ID per rule', done => { const testValue = 'testid'; - const xml = _generateSampleXml(`${testValue}` + - `${testValue}`); + const xml = _generateSampleXml(`${testValue}` + `${testValue}`); parseCorsXml(xml, log, err => { assert(err, 'Expected error but found none'); assert.strictEqual(err.is.MalformedXML, true); @@ -149,8 +138,7 @@ describe('PUT bucket cors :: helper validation functions ', () => { describe('validateMaxAgeSeconds ', () => { it('should validate successfully for valid value', done => { const testValue = 60; - const xml = _generateSampleXml(`${testValue}` + - ''); + const xml = _generateSampleXml(`${testValue}` + ''); parseCorsXml(xml, log, (err, result) => { assert.strictEqual(err, null, `Found unexpected err ${err}`); assert.strictEqual(typeof result[0].maxAgeSeconds, 'number'); @@ -159,12 +147,11 @@ describe('PUT bucket cors :: helper validation functions ', () => { }); }); - it('should return MalformedXML if more than one MaxAgeSeconds ' + - 'per rule', done => { + it('should return MalformedXML if more than one MaxAgeSeconds ' + 'per rule', done => { const testValue = '60'; const xml = _generateSampleXml( - `${testValue}` + - `${testValue}`); + `${testValue}` + `${testValue}`, + ); parseCorsXml(xml, log, err => { assert(err, 'Expected error but found none'); assert.strictEqual(err.is.MalformedXML, true); @@ -174,8 +161,7 @@ describe('PUT bucket cors :: helper validation functions ', () => { it('should validate & return undefined if empty value', done => { const testValue = ''; - const xml = _generateSampleXml(`${testValue}` + - ''); + const xml = _generateSampleXml(`${testValue}` + ''); parseCorsXml(xml, log, (err, result) => { assert.strictEqual(err, null, `Found unexpected err ${err}`); assert.strictEqual(result[0].MaxAgeSeconds, undefined); diff --git a/tests/unit/api/bucketPutEncryption.js b/tests/unit/api/bucketPutEncryption.js index b635973345..8fbae168f1 100644 --- a/tests/unit/api/bucketPutEncryption.js +++ b/tests/unit/api/bucketPutEncryption.js @@ -9,7 +9,6 @@ const bucketPutEncryption = require('../../../lib/api/bucketPutEncryption'); const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../helpers'); const { templateSSEConfig, templateRequest, getSSEConfig } = require('../utils/bucketEncryption'); - const log = new DummyRequestLogger(); const authInfo = makeAuthInfo('accessKey1'); const bucketName = 'bucketname'; @@ -44,26 +43,36 @@ describe('bucketPutEncryption API', () => { }); it('should reject a config with no Rule', done => { - bucketPutEncryption(authInfo, templateRequest(bucketName, - { post: ` + bucketPutEncryption( + authInfo, + templateRequest(bucketName, { + post: ` `, - }), log, err => { - assert.strictEqual(err.is.MalformedXML, true); - done(); - }); + }), + log, + err => { + assert.strictEqual(err.is.MalformedXML, true); + done(); + }, + ); }); it('should reject a config with no ApplyServerSideEncryptionByDefault section', done => { - bucketPutEncryption(authInfo, templateRequest(bucketName, - { post: ` + bucketPutEncryption( + authInfo, + templateRequest(bucketName, { + post: ` `, - }), log, err => { - assert.strictEqual(err.is.MalformedXML, true); - done(); - }); + }), + log, + err => { + assert.strictEqual(err.is.MalformedXML, true); + done(); + }, + ); }); it('should reject a config with no SSEAlgorithm', done => { @@ -170,8 +179,9 @@ describe('bucketPutEncryption API', () => { }); }); - it('should update SSEAlgorithm if existing SSEAlgorithm is AES256, ' + - 'new SSEAlgorithm is aws:kms and no KMSMasterKeyID is provided', + it( + 'should update SSEAlgorithm if existing SSEAlgorithm is AES256, ' + + 'new SSEAlgorithm is aws:kms and no KMSMasterKeyID is provided', done => { const post = templateSSEConfig({ algorithm: 'AES256' }); bucketPutEncryption(authInfo, templateRequest(bucketName, { post }), log, err => { @@ -180,7 +190,10 @@ describe('bucketPutEncryption API', () => { assert.ifError(err); const { masterKeyId } = sseInfo; const newConf = templateSSEConfig({ algorithm: 'aws:kms' }); - return bucketPutEncryption(authInfo, templateRequest(bucketName, { post: newConf }), log, + return bucketPutEncryption( + authInfo, + templateRequest(bucketName, { post: newConf }), + log, err => { assert.ifError(err); return getSSEConfig(bucketName, log, (err, updatedSSEInfo) => { @@ -192,11 +205,12 @@ describe('bucketPutEncryption API', () => { }); done(); }); - } + }, ); }); }); - }); + }, + ); it('should update SSEAlgorithm to aws:kms and set KMSMasterKeyID', done => { const post = templateSSEConfig({ algorithm: 'AES256' }); @@ -379,15 +393,13 @@ describe('bucketPutEncryption API with account level encryption', () => { assert.ifError(err); assert.deepStrictEqual(sseInfo, expectedSseInfo); const newConf = templateSSEConfig({ algorithm: 'AES256' }); - return bucketPutEncryption(authInfo, templateRequest(bucketName, { post: newConf }), log, - err => { - assert.ifError(err); - return getSSEConfig(bucketName, log, (err, updatedSSEInfo) => { - assert.deepStrictEqual(updatedSSEInfo, expectedSseInfo); - done(); - }); - } - ); + return bucketPutEncryption(authInfo, templateRequest(bucketName, { post: newConf }), log, err => { + assert.ifError(err); + return getSSEConfig(bucketName, log, (err, updatedSSEInfo) => { + assert.deepStrictEqual(updatedSSEInfo, expectedSseInfo); + done(); + }); + }); }); }); }); @@ -407,22 +419,20 @@ describe('bucketPutEncryption API with account level encryption', () => { }); const keyId = '12345'; const newConf = templateSSEConfig({ algorithm: 'aws:kms', keyId }); - return bucketPutEncryption(authInfo, templateRequest(bucketName, { post: newConf }), log, - err => { - assert.ifError(err); - return getSSEConfig(bucketName, log, (err, updatedSSEInfo) => { - assert.deepStrictEqual(updatedSSEInfo, { - cryptoScheme: 1, - algorithm: 'aws:kms', - mandatory: true, - masterKeyId: `${arnPrefix}${accountLevelMasterKeyId}`, - configuredMasterKeyId: `${arnPrefix}${keyId}`, - isAccountEncryptionEnabled: true, - }); - done(); + return bucketPutEncryption(authInfo, templateRequest(bucketName, { post: newConf }), log, err => { + assert.ifError(err); + return getSSEConfig(bucketName, log, (err, updatedSSEInfo) => { + assert.deepStrictEqual(updatedSSEInfo, { + cryptoScheme: 1, + algorithm: 'aws:kms', + mandatory: true, + masterKeyId: `${arnPrefix}${accountLevelMasterKeyId}`, + configuredMasterKeyId: `${arnPrefix}${keyId}`, + isAccountEncryptionEnabled: true, }); - } - ); + done(); + }); + }); }); }); }); @@ -441,21 +451,19 @@ describe('bucketPutEncryption API with account level encryption', () => { isAccountEncryptionEnabled: true, }); const newConf = templateSSEConfig({ algorithm: 'AES256' }); - return bucketPutEncryption(authInfo, templateRequest(bucketName, { post: newConf }), log, - err => { - assert.ifError(err); - return getSSEConfig(bucketName, log, (err, updatedSSEInfo) => { - assert.deepStrictEqual(updatedSSEInfo, { - cryptoScheme: 1, - algorithm: 'AES256', - mandatory: true, - masterKeyId: `${arnPrefix}${accountLevelMasterKeyId}`, - isAccountEncryptionEnabled: true, - }); - done(); + return bucketPutEncryption(authInfo, templateRequest(bucketName, { post: newConf }), log, err => { + assert.ifError(err); + return getSSEConfig(bucketName, log, (err, updatedSSEInfo) => { + assert.deepStrictEqual(updatedSSEInfo, { + cryptoScheme: 1, + algorithm: 'AES256', + mandatory: true, + masterKeyId: `${arnPrefix}${accountLevelMasterKeyId}`, + isAccountEncryptionEnabled: true, }); - } - ); + done(); + }); + }); }); }); }); @@ -474,21 +482,19 @@ describe('bucketPutEncryption API with account level encryption', () => { configuredMasterKeyId: `${arnPrefix}${keyId}`, }); const newConf = templateSSEConfig({ algorithm: 'AES256' }); - return bucketPutEncryption(authInfo, templateRequest(bucketName, { post: newConf }), log, - err => { - assert.ifError(err); - return getSSEConfig(bucketName, log, (err, updatedSSEInfo) => { - assert.deepStrictEqual(updatedSSEInfo, { - cryptoScheme: 1, - algorithm: 'AES256', - mandatory: true, - masterKeyId: `${arnPrefix}${accountLevelMasterKeyId}`, - isAccountEncryptionEnabled: true, - }); - done(); + return bucketPutEncryption(authInfo, templateRequest(bucketName, { post: newConf }), log, err => { + assert.ifError(err); + return getSSEConfig(bucketName, log, (err, updatedSSEInfo) => { + assert.deepStrictEqual(updatedSSEInfo, { + cryptoScheme: 1, + algorithm: 'AES256', + mandatory: true, + masterKeyId: `${arnPrefix}${accountLevelMasterKeyId}`, + isAccountEncryptionEnabled: true, }); - } - ); + done(); + }); + }); }); }); }); @@ -497,8 +503,9 @@ describe('bucketPutEncryption API with account level encryption', () => { describe('bucketPutEncryption API with failed vault service', () => { beforeEach(done => { sinon.stub(inMemory, 'supportsDefaultKeyPerAccount').value(true); - sinon.stub(vault, 'getOrCreateEncryptionKeyId').callsFake((accountCanonicalId, log, cb) => - cb(errors.ServiceFailure)); + sinon + .stub(vault, 'getOrCreateEncryptionKeyId') + .callsFake((accountCanonicalId, log, cb) => cb(errors.ServiceFailure)); bucketPut(authInfo, bucketPutRequest, log, done); }); diff --git a/tests/unit/api/bucketPutLifecycle.js b/tests/unit/api/bucketPutLifecycle.js index b3cd0071ec..2468cfe38b 100644 --- a/tests/unit/api/bucketPutLifecycle.js +++ b/tests/unit/api/bucketPutLifecycle.js @@ -2,12 +2,8 @@ const assert = require('assert'); const { bucketPut } = require('../../../lib/api/bucketPut'); const bucketPutLifecycle = require('../../../lib/api/bucketPutLifecycle'); -const { cleanup, - DummyRequestLogger, - makeAuthInfo } - = require('../helpers'); -const { getLifecycleRequest, getLifecycleXml } = - require('../utils/lifecycleHelpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../helpers'); +const { getLifecycleRequest, getLifecycleXml } = require('../utils/lifecycleHelpers'); const metadata = require('../../../lib/metadata/wrapper'); const log = new DummyRequestLogger(); @@ -82,9 +78,8 @@ describe('putBucketLifecycle API', () => { beforeEach(done => bucketPut(authInfo, testBucketPutRequest, log, done)); afterEach(() => cleanup()); - it('should update a bucket\'s metadata with lifecycle config obj', done => { - const testPutLifecycleRequest = getLifecycleRequest(bucketName, - getLifecycleXml()); + it("should update a bucket's metadata with lifecycle config obj", done => { + const testPutLifecycleRequest = getLifecycleRequest(bucketName, getLifecycleXml()); bucketPutLifecycle(authInfo, testPutLifecycleRequest, log, err => { if (err) { process.stdout.write(`Err putting lifecycle config ${err}`); @@ -95,10 +90,8 @@ describe('putBucketLifecycle API', () => { process.stdout.write(`Err retrieving bucket MD ${err}`); return done(err); } - const bucketLifecycleConfig = - bucket.getLifecycleConfiguration(); - assert.deepStrictEqual( - bucketLifecycleConfig, expectedLifecycleConfig); + const bucketLifecycleConfig = bucket.getLifecycleConfiguration(); + assert.deepStrictEqual(bucketLifecycleConfig, expectedLifecycleConfig); return done(); }); }); diff --git a/tests/unit/api/bucketPutLogging.js b/tests/unit/api/bucketPutLogging.js index 80b26c9a3d..8e81ccce5d 100644 --- a/tests/unit/api/bucketPutLogging.js +++ b/tests/unit/api/bucketPutLogging.js @@ -46,18 +46,22 @@ function createLoggingRequest(bucketName, post, headers = {}) { } function createValidLoggingXML(targetBucket, targetPrefix = 'logs/') { - return '' + + return ( + '' + '' + '' + `${targetBucket}` + `${targetPrefix}` + '' + - ''; + '' + ); } function createEmptyLoggingXML() { - return '' + - ''; + return ( + '' + + '' + ); } describe('bucketPutLogging API', () => { @@ -189,7 +193,8 @@ describe('bucketPutLogging API', () => { }); it('should return error for malformed XML - missing closing tag', done => { - const malformedXML = '' + + const malformedXML = + '' + '' + '' + `${targetBucket}` + @@ -206,7 +211,8 @@ describe('bucketPutLogging API', () => { }); it('should return error for malformed XML - invalid structure', done => { - const malformedXML = '' + + const malformedXML = + '' + '' + '' + 'invalid' + // Invalid tag @@ -234,7 +240,8 @@ describe('bucketPutLogging API', () => { }); it('should return NotImplemented error when TargetGrants is present', done => { - const loggingXMLWithGrants = '' + + const loggingXMLWithGrants = + '' + '' + '' + `${targetBucket}` + diff --git a/tests/unit/api/bucketPutNotification.js b/tests/unit/api/bucketPutNotification.js index 42456fba5b..cca311b78f 100644 --- a/tests/unit/api/bucketPutNotification.js +++ b/tests/unit/api/bucketPutNotification.js @@ -2,10 +2,7 @@ const assert = require('assert'); const { bucketPut } = require('../../../lib/api/bucketPut'); const bucketPutNotification = require('../../../lib/api/bucketPutNotification'); -const { cleanup, - DummyRequestLogger, - makeAuthInfo } - = require('../helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../helpers'); const metadata = require('../../../lib/metadata/wrapper'); const log = new DummyRequestLogger(); @@ -22,11 +19,7 @@ const expectedNotifConfig = { queueConfig: [ { id: 'notification-id', - events: [ - 's3:ObjectCreated:*', - 's3:ObjectTagging:*', - 's3:ObjectAcl:Put', - ], + events: ['s3:ObjectCreated:*', 's3:ObjectTagging:*', 's3:ObjectAcl:Put'], queueArn: 'arn:scality:bucketnotif:::target1', filterRules: undefined, }, @@ -34,16 +27,18 @@ const expectedNotifConfig = { }; function getNotifRequest(empty) { - const queueConfig = empty ? '' : - '' + - 'notification-id' + - 'arn:scality:bucketnotif:::target1' + - 's3:ObjectCreated:*' + - 's3:ObjectTagging:*' + - 's3:ObjectAcl:Put' + - ''; + const queueConfig = empty + ? '' + : '' + + 'notification-id' + + 'arn:scality:bucketnotif:::target1' + + 's3:ObjectCreated:*' + + 's3:ObjectTagging:*' + + 's3:ObjectAcl:Put' + + ''; - const notifXml = '' + + const notifXml = + '' + `${queueConfig}` + ''; diff --git a/tests/unit/api/bucketPutObjectLock.js b/tests/unit/api/bucketPutObjectLock.js index e048bb40f2..ca51f163fe 100644 --- a/tests/unit/api/bucketPutObjectLock.js +++ b/tests/unit/api/bucketPutObjectLock.js @@ -2,10 +2,7 @@ const assert = require('assert'); const { bucketPut } = require('../../../lib/api/bucketPut'); const bucketPutObjectLock = require('../../../lib/api/bucketPutObjectLock'); -const { cleanup, - DummyRequestLogger, - makeAuthInfo, -} = require('../helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../helpers'); const metadata = require('../../../lib/metadata/wrapper'); const log = new DummyRequestLogger(); @@ -18,7 +15,8 @@ const bucketPutRequest = { actionImplicitDenies: false, }; -const objectLockXml = '' + 'Enabled' + '' + @@ -57,21 +55,20 @@ describe('putBucketObjectLock API', () => { }); describe('with Object Lock enabled on bucket', () => { - const bucketObjLockRequest = Object.assign({}, bucketPutRequest, - { headers: { 'x-amz-bucket-object-lock-enabled': 'true' } }); + const bucketObjLockRequest = Object.assign({}, bucketPutRequest, { + headers: { 'x-amz-bucket-object-lock-enabled': 'true' }, + }); beforeEach(done => bucketPut(authInfo, bucketObjLockRequest, log, done)); afterEach(() => cleanup()); - it('should update a bucket\'s metadata with object lock config', done => { + it("should update a bucket's metadata with object lock config", done => { bucketPutObjectLock(authInfo, putObjLockRequest, log, err => { assert.ifError(err); return metadata.getBucket(bucketName, log, (err, bucket) => { assert.ifError(err); - const bucketObjectLockConfig = bucket. - getObjectLockConfiguration(); - assert.deepStrictEqual( - bucketObjectLockConfig, expectedObjectLockConfig); + const bucketObjectLockConfig = bucket.getObjectLockConfiguration(); + assert.deepStrictEqual(bucketObjectLockConfig, expectedObjectLockConfig); return done(); }); }); diff --git a/tests/unit/api/bucketPutPolicy.js b/tests/unit/api/bucketPutPolicy.js index b135ef09f7..c3ef92e2a9 100644 --- a/tests/unit/api/bucketPutPolicy.js +++ b/tests/unit/api/bucketPutPolicy.js @@ -2,10 +2,7 @@ const assert = require('assert'); const { bucketPut } = require('../../../lib/api/bucketPut'); const bucketPutPolicy = require('../../../lib/api/bucketPutPolicy'); -const { cleanup, - DummyRequestLogger, - makeAuthInfo } - = require('../helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../helpers'); const metadata = require('../../../lib/metadata/wrapper'); const log = new DummyRequestLogger(); @@ -48,9 +45,8 @@ describe('putBucketPolicy API', () => { }); afterEach(() => cleanup()); - it('should update a bucket\'s metadata with bucket policy obj', done => { - bucketPutPolicy(authInfo, getPolicyRequest(expectedBucketPolicy), - log, err => { + it("should update a bucket's metadata with bucket policy obj", done => { + bucketPutPolicy(authInfo, getPolicyRequest(expectedBucketPolicy), log, err => { if (err) { process.stdout.write(`Err putting bucket policy ${err}`); return done(err); @@ -67,11 +63,9 @@ describe('putBucketPolicy API', () => { }); }); - it('should return error if policy resource does not include bucket name', - done => { + it('should return error if policy resource does not include bucket name', done => { expectedBucketPolicy.Statement[0].Resource = 'arn:aws::s3:::badname'; - bucketPutPolicy(authInfo, getPolicyRequest(expectedBucketPolicy), - log, err => { + bucketPutPolicy(authInfo, getPolicyRequest(expectedBucketPolicy), log, err => { assert.strictEqual(err.is.MalformedPolicy, true); assert.strictEqual(err.description, 'Policy has invalid resource'); return done(); @@ -79,10 +73,8 @@ describe('putBucketPolicy API', () => { }); it('should not return error if policy contains conditions', done => { - expectedBucketPolicy.Statement[0].Condition = - { IpAddress: { 'aws:SourceIp': '123.123.123.123' } }; - bucketPutPolicy(authInfo, getPolicyRequest(expectedBucketPolicy), log, - err => { + expectedBucketPolicy.Statement[0].Condition = { IpAddress: { 'aws:SourceIp': '123.123.123.123' } }; + bucketPutPolicy(authInfo, getPolicyRequest(expectedBucketPolicy), log, err => { assert.ifError(err); done(); }); @@ -90,18 +82,15 @@ describe('putBucketPolicy API', () => { it('should return error if policy contains service principal', done => { expectedBucketPolicy.Statement[0].Principal = { Service: ['test.com'] }; - bucketPutPolicy(authInfo, getPolicyRequest(expectedBucketPolicy), log, - err => { + bucketPutPolicy(authInfo, getPolicyRequest(expectedBucketPolicy), log, err => { assert.strictEqual(err.is.NotImplemented, true); done(); }); }); it('should return error if policy contains federated principal', done => { - expectedBucketPolicy.Statement[0].Principal = - { Federated: 'www.test.com' }; - bucketPutPolicy(authInfo, getPolicyRequest(expectedBucketPolicy), log, - err => { + expectedBucketPolicy.Statement[0].Principal = { Federated: 'www.test.com' }; + bucketPutPolicy(authInfo, getPolicyRequest(expectedBucketPolicy), log, err => { assert.strictEqual(err.is.NotImplemented, true); done(); }); diff --git a/tests/unit/api/bucketPutReplication.js b/tests/unit/api/bucketPutReplication.js index 4a19f5e24b..90b9fc4b84 100644 --- a/tests/unit/api/bucketPutReplication.js +++ b/tests/unit/api/bucketPutReplication.js @@ -1,14 +1,10 @@ const assert = require('assert'); -const bucketPutReplication = - require('../../../lib/api/bucketPutReplication'); +const bucketPutReplication = require('../../../lib/api/bucketPutReplication'); const { DummyRequestLogger, makeAuthInfo } = require('../helpers'); -const { getReplicationConfiguration } = - require('../../../lib/api/apiUtils/bucket/getReplicationConfiguration'); -const validateReplicationConfig = - require('../../../lib/api/apiUtils/bucket/validateReplicationConfig'); -const replicationUtils = - require('../../functional/aws-node-sdk/lib/utility/replication'); +const { getReplicationConfiguration } = require('../../../lib/api/apiUtils/bucket/getReplicationConfiguration'); +const validateReplicationConfig = require('../../../lib/api/apiUtils/bucket/validateReplicationConfig'); +const replicationUtils = require('../../functional/aws-node-sdk/lib/utility/replication'); const log = new DummyRequestLogger(); // Check for the expected error response code and status code. @@ -17,8 +13,10 @@ function checkError(xml, expectedErr, cb) { if (expectedErr === null) { assert.strictEqual(err, null, `expected no error but got '${err}'`); } else { - assert(err.is[expectedErr], 'incorrect error response: should be ' + - `'Error: ${expectedErr}' but got '${err}'`); + assert( + err.is[expectedErr], + 'incorrect error response: should be ' + `'Error: ${expectedErr}' but got '${err}'`, + ); } return cb(); }); @@ -31,45 +29,42 @@ function checkGeneratedID(xml, cb) { return cb(err); } const id = res.rules[0].id; - assert.strictEqual(typeof id, 'string', 'expected rule ID to be ' + - `string but got ${typeof id}`); - assert.strictEqual(id.length, 48, 'expected rule ID to be a length ' + - `of 48 but got ${id.length}`); + assert.strictEqual(typeof id, 'string', 'expected rule ID to be ' + `string but got ${typeof id}`); + assert.strictEqual(id.length, 48, 'expected rule ID to be a length ' + `of 48 but got ${id.length}`); return cb(); }); } // Create replication configuration XML with an tag optionally omitted. function createReplicationXML(missingTag, tagValue) { - let Role = missingTag === 'Role' ? '' : - '' + - 'arn:aws:iam::account-id:role/src-resource,' + - 'arn:aws:iam::account-id:role/dest-resource' + - ''; + let Role = + missingTag === 'Role' + ? '' + : '' + + 'arn:aws:iam::account-id:role/src-resource,' + + 'arn:aws:iam::account-id:role/dest-resource' + + ''; Role = tagValue && tagValue.Role ? `${tagValue.Role}` : Role; let ID = missingTag === 'ID' ? '' : 'foo'; ID = tagValue && tagValue.ID === '' ? '' : ID; const Prefix = missingTag === 'Prefix' ? '' : 'foo'; const Status = missingTag === 'Status' ? '' : 'Enabled'; - const Bucket = missingTag === 'Bucket' ? '' : - 'arn:aws:s3:::destination-bucket'; - let StorageClass = missingTag === 'StorageClass' ? '' : - 'STANDARD'; - StorageClass = tagValue && tagValue.StorageClass ? - `${tagValue.StorageClass}` : StorageClass; - const Destination = missingTag === 'Destination' ? '' : - `${Bucket + StorageClass}`; - const Rule = missingTag === 'Rule' ? '' : - `${ID + Prefix + Status + Destination}`; + const Bucket = missingTag === 'Bucket' ? '' : 'arn:aws:s3:::destination-bucket'; + let StorageClass = missingTag === 'StorageClass' ? '' : 'STANDARD'; + StorageClass = + tagValue && tagValue.StorageClass ? `${tagValue.StorageClass}` : StorageClass; + const Destination = missingTag === 'Destination' ? '' : `${Bucket + StorageClass}`; + const Rule = missingTag === 'Rule' ? '' : `${ID + Prefix + Status + Destination}`; const content = missingTag === null ? '' : `${Role}${Rule}`; - return '${content}` + - ''; + return ( + '${content}` + + '' + ); } -describe('\'getReplicationConfiguration\' function', () => { - it('should not return error when putting valid XML', done => - checkError(createReplicationXML(), null, done)); +describe("'getReplicationConfiguration' function", () => { + it('should not return error when putting valid XML', done => checkError(createReplicationXML(), null, done)); it('should not accept empty replication configuration', done => checkError(createReplicationXML(null), 'MalformedXML', done)); @@ -79,13 +74,13 @@ describe('\'getReplicationConfiguration\' function', () => { const xmlTag = prop === 'Rules' ? 'Rule' : prop; const xml = createReplicationXML(xmlTag); - it(`should not accept replication configuration without '${prop}'`, - done => checkError(xml, 'MalformedXML', done)); + it(`should not accept replication configuration without '${prop}'`, done => + checkError(xml, 'MalformedXML', done)); }); replicationUtils.optionalConfigProperties.forEach(prop => { - it(`should accept replication configuration without '${prop}'`, - done => checkError(createReplicationXML(prop), null, done)); + it(`should accept replication configuration without '${prop}'`, done => + checkError(createReplicationXML(prop), null, done)); }); it(`should accept replication configuration without 'Bucket' when there @@ -97,19 +92,18 @@ describe('\'getReplicationConfiguration\' function', () => { checkError(xml, null, done); }); - it("should create a rule 'ID' if omitted from the replication " + - 'configuration', done => { + it("should create a rule 'ID' if omitted from the replication " + 'configuration', done => { const xml = createReplicationXML('ID'); return checkGeneratedID(xml, done); }); - it('should create an \'ID\' if rule ID is \'\'', done => { + it("should create an 'ID' if rule ID is ''", done => { const xml = createReplicationXML(undefined, { ID: '' }); return checkGeneratedID(xml, done); }); }); -describe('\'validateReplicationConfig\' function', () => { +describe("'validateReplicationConfig' function", () => { const nonTransientBucket = { getLocationConstraint: () => 'us-east-1', }; @@ -117,59 +111,67 @@ describe('\'validateReplicationConfig\' function', () => { getLocationConstraint: () => 'transientfile', }; - it('should validate configuration when bucket location is ' + - 'not transient and preferred read location is not specified', () => { - const withoutPreferredRead = { - role: 'arn:aws:iam::account-id:role/src-resource,' + - 'arn:aws:iam::account-id:role/dest-resource', - destination: 'arn:aws:s3:::destination-bucket', - rules: [{ - prefix: 'test-prefix', - enabled: true, - id: 'test-id', - storageClass: 'STANDARD,us-east-2', - }], - }; - const result = validateReplicationConfig(withoutPreferredRead, - nonTransientBucket); - assert.strictEqual(result, true); - }); - - it('should validate configuration when bucket location is transient ' + - 'and preferred read location is specified', () => { - const withPreferredRead = { - role: 'arn:aws:iam::account-id:role/src-resource,' + - 'arn:aws:iam::account-id:role/dest-resource', - destination: 'arn:aws:s3:::destination-bucket', - rules: [{ - prefix: 'test-prefix', - enabled: true, - id: 'test-id', - storageClass: 'STANDARD,us-east-2:preferred_read', - }], - }; - const result = validateReplicationConfig(withPreferredRead, - transientBucket); - assert.strictEqual(result, true); - }); - - it('should not validate configuration when bucket location is ' + - 'transient and preferred read location is not specified', () => { - const withoutPreferredRead = { - role: 'arn:aws:iam::account-id:role/src-resource,' + - 'arn:aws:iam::account-id:role/dest-resource', - destination: 'arn:aws:s3:::destination-bucket', - rules: [{ - prefix: 'test-prefix', - enabled: true, - id: 'test-id', - storageClass: 'STANDARD,us-east-2', - }], - }; - const result = validateReplicationConfig(withoutPreferredRead, - transientBucket); - assert.strictEqual(result, false); - }); + it( + 'should validate configuration when bucket location is ' + + 'not transient and preferred read location is not specified', + () => { + const withoutPreferredRead = { + role: 'arn:aws:iam::account-id:role/src-resource,' + 'arn:aws:iam::account-id:role/dest-resource', + destination: 'arn:aws:s3:::destination-bucket', + rules: [ + { + prefix: 'test-prefix', + enabled: true, + id: 'test-id', + storageClass: 'STANDARD,us-east-2', + }, + ], + }; + const result = validateReplicationConfig(withoutPreferredRead, nonTransientBucket); + assert.strictEqual(result, true); + }, + ); + + it( + 'should validate configuration when bucket location is transient ' + 'and preferred read location is specified', + () => { + const withPreferredRead = { + role: 'arn:aws:iam::account-id:role/src-resource,' + 'arn:aws:iam::account-id:role/dest-resource', + destination: 'arn:aws:s3:::destination-bucket', + rules: [ + { + prefix: 'test-prefix', + enabled: true, + id: 'test-id', + storageClass: 'STANDARD,us-east-2:preferred_read', + }, + ], + }; + const result = validateReplicationConfig(withPreferredRead, transientBucket); + assert.strictEqual(result, true); + }, + ); + + it( + 'should not validate configuration when bucket location is ' + + 'transient and preferred read location is not specified', + () => { + const withoutPreferredRead = { + role: 'arn:aws:iam::account-id:role/src-resource,' + 'arn:aws:iam::account-id:role/dest-resource', + destination: 'arn:aws:s3:::destination-bucket', + rules: [ + { + prefix: 'test-prefix', + enabled: true, + id: 'test-id', + storageClass: 'STANDARD,us-east-2', + }, + ], + }; + const result = validateReplicationConfig(withoutPreferredRead, transientBucket); + assert.strictEqual(result, false); + }, + ); }); describe('bucketPutReplication API', () => { diff --git a/tests/unit/api/bucketPutTagging.js b/tests/unit/api/bucketPutTagging.js index 1d95729c46..4b6324b078 100644 --- a/tests/unit/api/bucketPutTagging.js +++ b/tests/unit/api/bucketPutTagging.js @@ -1,11 +1,7 @@ const assert = require('assert'); const { bucketPut } = require('../../../lib/api/bucketPut'); -const { cleanup, - DummyRequestLogger, - makeAuthInfo, - TaggingConfigTester, -} = require('../helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo, TaggingConfigTester } = require('../helpers'); const bucketPutTagging = require('../../../lib/api/bucketPutTagging'); const log = new DummyRequestLogger(); const authInfo = makeAuthInfo('accessKey1'); @@ -28,8 +24,7 @@ describe('putBucketTagging API', () => { it('should set tags resource', done => { const taggingUtil = new TaggingConfigTester(); - const testBucketPutTaggingRequest = taggingUtil - .createBucketTaggingRequest('PUT', bucketName); + const testBucketPutTaggingRequest = taggingUtil.createBucketTaggingRequest('PUT', bucketName); bucketPutTagging(authInfo, testBucketPutTaggingRequest, log, err => { if (err) { process.stdout.write(`Err putting object tagging ${err}`); @@ -42,8 +37,7 @@ describe('putBucketTagging API', () => { it('should return access denied if the authorization check fails', done => { const taggingUtil = new TaggingConfigTester(); - const testBucketPutTaggingRequest = taggingUtil - .createBucketTaggingRequest('PUT', bucketName); + const testBucketPutTaggingRequest = taggingUtil.createBucketTaggingRequest('PUT', bucketName); const authInfo = makeAuthInfo('accessKey2'); bucketPutTagging(authInfo, testBucketPutTaggingRequest, log, err => { assert(err.AccessDenied); diff --git a/tests/unit/api/bucketPutVersioning.js b/tests/unit/api/bucketPutVersioning.js index 462fadcf74..7cb4ef24f0 100644 --- a/tests/unit/api/bucketPutVersioning.js +++ b/tests/unit/api/bucketPutVersioning.js @@ -6,46 +6,44 @@ const { bucketPut } = require('../../../lib/api/bucketPut'); const bucketPutVersioning = require('../../../lib/api/bucketPutVersioning'); const bucketPutReplication = require('../../../lib/api/bucketPutReplication'); -const { cleanup, - DummyRequestLogger, - makeAuthInfo } = require('../helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../helpers'); const metadata = require('../../../lib/metadata/wrapper'); const xmlEnableVersioning = -'' + -'Enabled' + -''; + '' + + 'Enabled' + + ''; const xmlSuspendVersioning = -'' + -'Suspended' + -''; + '' + + 'Suspended' + + ''; const locConstraintVersioned = -'' + -'withversioning' + -''; + '' + + 'withversioning' + + ''; const locConstraintNonVersioned = -'' + -'withoutversioning' + -''; + '' + + 'withoutversioning' + + ''; const xmlReplicationConfiguration = -'' + + '' + 'arn:aws:iam::account-id:role/src-resource' + '' + - '' + - 'Enabled' + - '' + - 'arn:aws:s3:::destination-bucket' + - 'us-east-2' + - '' + + '' + + 'Enabled' + + '' + + 'arn:aws:s3:::destination-bucket' + + 'us-east-2' + + '' + '' + -''; + ''; -const externalVersioningErrorMessage = 'We do not currently support putting ' + -'a versioned object to a location-constraint of type Azure or GCP.'; +const externalVersioningErrorMessage = + 'We do not currently support putting ' + 'a versioned object to a location-constraint of type Azure or GCP.'; const log = new DummyRequestLogger(); const bucketName = 'bucketname'; @@ -94,55 +92,58 @@ describe('bucketPutVersioning API', () => { const tests = [ { - msg: 'should successfully enable versioning on location ' + - 'constraint with supportsVersioning set to true', + msg: + 'should successfully enable versioning on location ' + + 'constraint with supportsVersioning set to true', input: xmlEnableVersioning, output: { Status: 'Enabled' }, }, { - msg: 'should successfully suspend versioning on location ' + - 'constraint with supportsVersioning set to true', + msg: + 'should successfully suspend versioning on location ' + + 'constraint with supportsVersioning set to true', input: xmlSuspendVersioning, output: { Status: 'Suspended' }, }, ]; - tests.forEach(test => it(test.msg, done => { - const request = _putVersioningRequest(test.input); - bucketPutVersioning(authInfo, request, log, err => { - assert.ifError(err, - `Expected success, but got err: ${err}`); - metadata.getBucket(bucketName, log, (err, bucket) => { - assert.ifError(err, - `Expected success, but got err: ${err}`); - assert.deepStrictEqual(bucket._versioningConfiguration, - test.output); - done(); + tests.forEach(test => + it(test.msg, done => { + const request = _putVersioningRequest(test.input); + bucketPutVersioning(authInfo, request, log, err => { + assert.ifError(err, `Expected success, but got err: ${err}`); + metadata.getBucket(bucketName, log, (err, bucket) => { + assert.ifError(err, `Expected success, but got err: ${err}`); + assert.deepStrictEqual(bucket._versioningConfiguration, test.output); + done(); + }); }); - }); - })); + }), + ); it('should not suspend versioning on bucket with replication', done => { - async.series([ - // Enable versioning to allow putting a replication config. - next => { - const request = _putVersioningRequest(xmlEnableVersioning); - bucketPutVersioning(authInfo, request, log, next); - }, - // Put the replication config on the bucket. - next => { - const request = - _putReplicationRequest(xmlReplicationConfiguration); - bucketPutReplication(authInfo, request, log, next); - }, - // Attempt to suspend versioning. - next => { - const request = _putVersioningRequest(xmlSuspendVersioning); - bucketPutVersioning(authInfo, request, log, err => { - assert(err.is.InvalidBucketState); - next(); - }); - }, - ], done); + async.series( + [ + // Enable versioning to allow putting a replication config. + next => { + const request = _putVersioningRequest(xmlEnableVersioning); + bucketPutVersioning(authInfo, request, log, next); + }, + // Put the replication config on the bucket. + next => { + const request = _putReplicationRequest(xmlReplicationConfiguration); + bucketPutReplication(authInfo, request, log, next); + }, + // Attempt to suspend versioning. + next => { + const request = _putVersioningRequest(xmlSuspendVersioning); + bucketPutVersioning(authInfo, request, log, err => { + assert(err.is.InvalidBucketState); + next(); + }); + }, + ], + done, + ); }); }); @@ -154,28 +155,28 @@ describe('bucketPutVersioning API', () => { const tests = [ { - msg: 'should return error if enabling versioning on location ' + - 'constraint with supportsVersioning set to false', + msg: + 'should return error if enabling versioning on location ' + + 'constraint with supportsVersioning set to false', input: xmlEnableVersioning, - output: { error: errorInstances.NotImplemented.customizeDescription( - externalVersioningErrorMessage) }, + output: { error: errorInstances.NotImplemented.customizeDescription(externalVersioningErrorMessage) }, }, { - msg: 'should return error if suspending versioning on ' + - ' location constraint with supportsVersioning set to false', + msg: + 'should return error if suspending versioning on ' + + ' location constraint with supportsVersioning set to false', input: xmlSuspendVersioning, - output: { error: errorInstances.NotImplemented.customizeDescription( - externalVersioningErrorMessage) }, + output: { error: errorInstances.NotImplemented.customizeDescription(externalVersioningErrorMessage) }, }, ]; - tests.forEach(test => it(test.msg, done => { - const putBucketVersioningRequest = - _putVersioningRequest(test.input); - bucketPutVersioning(authInfo, putBucketVersioningRequest, log, - err => { - assert.deepStrictEqual(err, test.output.error); - done(); - }); - })); + tests.forEach(test => + it(test.msg, done => { + const putBucketVersioningRequest = _putVersioningRequest(test.input); + bucketPutVersioning(authInfo, putBucketVersioningRequest, log, err => { + assert.deepStrictEqual(err, test.output.error); + done(); + }); + }), + ); }); }); diff --git a/tests/unit/api/bucketPutWebsite.js b/tests/unit/api/bucketPutWebsite.js index 45944910af..c2f46ee62f 100644 --- a/tests/unit/api/bucketPutWebsite.js +++ b/tests/unit/api/bucketPutWebsite.js @@ -3,13 +3,8 @@ const { parseString } = require('xml2js'); const { bucketPut } = require('../../../lib/api/bucketPut'); const bucketPutWebsite = require('../../../lib/api/bucketPutWebsite'); -const { xmlContainsElem } - = require('../../../lib/api/apiUtils/bucket/bucketWebsite'); -const { cleanup, - DummyRequestLogger, - makeAuthInfo, - WebsiteConfig } - = require('../helpers'); +const { xmlContainsElem } = require('../../../lib/api/apiUtils/bucket/bucketWebsite'); +const { cleanup, DummyRequestLogger, makeAuthInfo, WebsiteConfig } = require('../helpers'); const metadata = require('../../../lib/metadata/wrapper'); const log = new DummyRequestLogger(); @@ -41,12 +36,10 @@ describe('putBucketWebsite API', () => { beforeEach(done => bucketPut(authInfo, testBucketPutRequest, log, done)); afterEach(() => cleanup()); - it('should update a bucket\'s metadata with website config obj', done => { + it("should update a bucket's metadata with website config obj", done => { const config = new WebsiteConfig('index.html', 'error.html'); - config.addRoutingRule({ ReplaceKeyPrefixWith: 'documents/' }, - { KeyPrefixEquals: 'docs/' }); - const testBucketPutWebsiteRequest = - _getPutWebsiteRequest(config.getXml()); + config.addRoutingRule({ ReplaceKeyPrefixWith: 'documents/' }, { KeyPrefixEquals: 'docs/' }); + const testBucketPutWebsiteRequest = _getPutWebsiteRequest(config.getXml()); bucketPutWebsite(authInfo, testBucketPutWebsiteRequest, log, err => { if (err) { process.stdout.write(`Err putting website config ${err}`); @@ -58,122 +51,122 @@ describe('putBucketWebsite API', () => { return done(err); } const bucketWebsiteConfig = bucket.getWebsiteConfiguration(); - assert.strictEqual(bucketWebsiteConfig._indexDocument, - config.IndexDocument.Suffix); - assert.strictEqual(bucketWebsiteConfig._errorDocument, - config.ErrorDocument.Key); - assert.strictEqual(bucketWebsiteConfig._routingRules[0] - ._condition.keyPrefixEquals, - config.RoutingRules[0].Condition.KeyPrefixEquals); - assert.strictEqual(bucketWebsiteConfig._routingRules[0] - ._redirect.replaceKeyPrefixWith, - config.RoutingRules[0].Redirect.ReplaceKeyPrefixWith); + assert.strictEqual(bucketWebsiteConfig._indexDocument, config.IndexDocument.Suffix); + assert.strictEqual(bucketWebsiteConfig._errorDocument, config.ErrorDocument.Key); + assert.strictEqual( + bucketWebsiteConfig._routingRules[0]._condition.keyPrefixEquals, + config.RoutingRules[0].Condition.KeyPrefixEquals, + ); + assert.strictEqual( + bucketWebsiteConfig._routingRules[0]._redirect.replaceKeyPrefixWith, + config.RoutingRules[0].Redirect.ReplaceKeyPrefixWith, + ); return done(); }); }); }); describe('helper functions', () => { - it('xmlContainsElem should return true if xml contains ' + - 'specified element', done => { - const xml = '' + - 'value' + - ''; + it('xmlContainsElem should return true if xml contains ' + 'specified element', done => { + const xml = '' + 'value' + ''; parseString(xml, (err, result) => { if (err) { process.stdout.write(`Unexpected err ${err} parsing xml`); return done(err); } - const containsRes = xmlContainsElem(result.Toplevel.Parent, - 'Element'); + const containsRes = xmlContainsElem(result.Toplevel.Parent, 'Element'); assert.strictEqual(containsRes, true); return done(); }); }); - it('xmlContainsElem should return false if xml does not contain ' + - 'specified element', done => { - const xml = '' + - 'value' + - ''; + it('xmlContainsElem should return false if xml does not contain ' + 'specified element', done => { + const xml = '' + 'value' + ''; parseString(xml, (err, result) => { if (err) { process.stdout.write(`Unexpected err ${err} parsing xml`); return done(err); } - const containsRes = xmlContainsElem(result.Toplevel.Parent, - 'Element'); + const containsRes = xmlContainsElem(result.Toplevel.Parent, 'Element'); assert.strictEqual(containsRes, false); return done(); }); }); - it('xmlContainsElem should return true if parent contains list of ' + - 'elements and isList is specified in options', done => { - const xml = '' + - 'value' + - 'value' + - 'value' + - ''; - parseString(xml, (err, result) => { - if (err) { - process.stdout.write(`Unexpected err ${err} parsing xml`); - return done(err); - } - const containsRes = xmlContainsElem(result.Toplevel.Parent, - 'Element', { isList: true }); - assert.strictEqual(containsRes, true); - return done(); - }); - }); - it('xmlContainsElem should return true if parent contains at least ' + - 'one of the elements specified, if multiple', done => { - const xml = '' + - 'value' + - ''; - parseString(xml, (err, result) => { - if (err) { - process.stdout.write(`Unexpected err ${err} parsing xml`); - return done(err); - } - const containsRes = xmlContainsElem(result.Toplevel.Parent, - ['ElementA', 'ElementB']); - assert.strictEqual(containsRes, true); - return done(); - }); - }); - it('xmlContainsElem should return false if parent contains only one ' + - 'of multiple elements specified and checkForAll specified in options', - done => { - const xml = '' + - 'value' + - ''; - parseString(xml, (err, result) => { - if (err) { - process.stdout.write(`Unexpected err ${err} parsing xml`); - return done(err); - } - const containsRes = xmlContainsElem(result.Toplevel.Parent, - ['ElementA', 'ElementB'], { checkForAll: true }); - assert.strictEqual(containsRes, false); - return done(); - }); - }); - it('xmlContainsElem should return true if parent contains all ' + - 'of multiple elements specified and checkForAll specified in options', - done => { - const xml = '' + - 'value' + - 'value' + - ''; - parseString(xml, (err, result) => { - if (err) { - process.stdout.write(`Unexpected err ${err} parsing xml`); - return done(err); - } - const containsRes = xmlContainsElem(result.Toplevel.Parent, - ['ElementA', 'ElementB'], { checkForAll: true }); - assert.strictEqual(containsRes, true); - return done(); - }); - }); + it( + 'xmlContainsElem should return true if parent contains list of ' + + 'elements and isList is specified in options', + done => { + const xml = + '' + + 'value' + + 'value' + + 'value' + + ''; + parseString(xml, (err, result) => { + if (err) { + process.stdout.write(`Unexpected err ${err} parsing xml`); + return done(err); + } + const containsRes = xmlContainsElem(result.Toplevel.Parent, 'Element', { isList: true }); + assert.strictEqual(containsRes, true); + return done(); + }); + }, + ); + it( + 'xmlContainsElem should return true if parent contains at least ' + + 'one of the elements specified, if multiple', + done => { + const xml = '' + 'value' + ''; + parseString(xml, (err, result) => { + if (err) { + process.stdout.write(`Unexpected err ${err} parsing xml`); + return done(err); + } + const containsRes = xmlContainsElem(result.Toplevel.Parent, ['ElementA', 'ElementB']); + assert.strictEqual(containsRes, true); + return done(); + }); + }, + ); + it( + 'xmlContainsElem should return false if parent contains only one ' + + 'of multiple elements specified and checkForAll specified in options', + done => { + const xml = '' + 'value' + ''; + parseString(xml, (err, result) => { + if (err) { + process.stdout.write(`Unexpected err ${err} parsing xml`); + return done(err); + } + const containsRes = xmlContainsElem(result.Toplevel.Parent, ['ElementA', 'ElementB'], { + checkForAll: true, + }); + assert.strictEqual(containsRes, false); + return done(); + }); + }, + ); + it( + 'xmlContainsElem should return true if parent contains all ' + + 'of multiple elements specified and checkForAll specified in options', + done => { + const xml = + '' + + 'value' + + 'value' + + ''; + parseString(xml, (err, result) => { + if (err) { + process.stdout.write(`Unexpected err ${err} parsing xml`); + return done(err); + } + const containsRes = xmlContainsElem(result.Toplevel.Parent, ['ElementA', 'ElementB'], { + checkForAll: true, + }); + assert.strictEqual(containsRes, true); + return done(); + }); + }, + ); }); }); diff --git a/tests/unit/api/corsErrorHeaders.js b/tests/unit/api/corsErrorHeaders.js index 89216f4cdb..102a9f58eb 100644 --- a/tests/unit/api/corsErrorHeaders.js +++ b/tests/unit/api/corsErrorHeaders.js @@ -10,55 +10,42 @@ const { bucketPut } = require('../../../lib/api/bucketPut'); const bucketPutCors = require('../../../lib/api/bucketPutCors'); const metadata = require('../../../lib/metadata/wrapper'); const DummyRequest = require('../DummyRequest'); -const { - CorsConfigTester, - DummyRequestLogger, - cleanup, - makeAuthInfo, -} = require('../helpers'); +const { CorsConfigTester, DummyRequestLogger, cleanup, makeAuthInfo } = require('../helpers'); const endpoints = [ { apiMethod: 'bucketGet', httpMethod: 'GET', url: '/', query: {} }, { apiMethod: 'bucketHead', httpMethod: 'HEAD', url: '/', query: {} }, { apiMethod: 'bucketDelete', httpMethod: 'DELETE', url: '/', query: {} }, - { apiMethod: 'bucketGetACL', httpMethod: 'GET', url: '/?acl', - query: { acl: '' } }, - { apiMethod: 'bucketGetCors', httpMethod: 'GET', url: '/?cors', - query: { cors: '' } }, - { apiMethod: 'bucketGetLifecycle', httpMethod: 'GET', url: '/?lifecycle', - query: { lifecycle: '' } }, - { apiMethod: 'bucketGetReplication', httpMethod: 'GET', - url: '/?replication', query: { replication: '' } }, - { apiMethod: 'bucketGetPolicy', httpMethod: 'GET', url: '/?policy', - query: { policy: '' } }, - { apiMethod: 'bucketGetVersioning', httpMethod: 'GET', url: '/?versioning', - query: { versioning: '' } }, - { apiMethod: 'bucketGetWebsite', httpMethod: 'GET', url: '/?website', - query: { website: '' } }, - { apiMethod: 'bucketGetTagging', httpMethod: 'GET', url: '/?tagging', - query: { tagging: '' } }, - { apiMethod: 'bucketGetEncryption', httpMethod: 'GET', url: '/?encryption', - query: { encryption: '' } }, - { apiMethod: 'bucketGetNotification', httpMethod: 'GET', - url: '/?notification', query: { notification: '' } }, - { apiMethod: 'bucketGetObjectLock', httpMethod: 'GET', - url: '/?object-lock', query: { 'object-lock': '' } }, - { apiMethod: 'bucketGetLocation', httpMethod: 'GET', url: '/?location', - query: { location: '' } }, - { apiMethod: 'objectGet', httpMethod: 'GET', url: '/obj', query: {}, - objectKey: 'obj' }, - { apiMethod: 'objectHead', httpMethod: 'HEAD', url: '/obj', query: {}, - objectKey: 'obj' }, - { apiMethod: 'objectDelete', httpMethod: 'DELETE', url: '/obj', query: {}, - objectKey: 'obj' }, - { apiMethod: 'objectGetLegalHold', httpMethod: 'GET', - url: '/obj?legal-hold', query: { 'legal-hold': '' }, - objectKey: 'obj' }, - { apiMethod: 'objectGetAttributes', httpMethod: 'GET', - url: '/obj?attributes', query: { attributes: '' }, - objectKey: 'obj' }, - { apiMethod: 'listMultipartUploads', httpMethod: 'GET', url: '/?uploads', - query: { uploads: '' } }, + { apiMethod: 'bucketGetACL', httpMethod: 'GET', url: '/?acl', query: { acl: '' } }, + { apiMethod: 'bucketGetCors', httpMethod: 'GET', url: '/?cors', query: { cors: '' } }, + { apiMethod: 'bucketGetLifecycle', httpMethod: 'GET', url: '/?lifecycle', query: { lifecycle: '' } }, + { apiMethod: 'bucketGetReplication', httpMethod: 'GET', url: '/?replication', query: { replication: '' } }, + { apiMethod: 'bucketGetPolicy', httpMethod: 'GET', url: '/?policy', query: { policy: '' } }, + { apiMethod: 'bucketGetVersioning', httpMethod: 'GET', url: '/?versioning', query: { versioning: '' } }, + { apiMethod: 'bucketGetWebsite', httpMethod: 'GET', url: '/?website', query: { website: '' } }, + { apiMethod: 'bucketGetTagging', httpMethod: 'GET', url: '/?tagging', query: { tagging: '' } }, + { apiMethod: 'bucketGetEncryption', httpMethod: 'GET', url: '/?encryption', query: { encryption: '' } }, + { apiMethod: 'bucketGetNotification', httpMethod: 'GET', url: '/?notification', query: { notification: '' } }, + { apiMethod: 'bucketGetObjectLock', httpMethod: 'GET', url: '/?object-lock', query: { 'object-lock': '' } }, + { apiMethod: 'bucketGetLocation', httpMethod: 'GET', url: '/?location', query: { location: '' } }, + { apiMethod: 'objectGet', httpMethod: 'GET', url: '/obj', query: {}, objectKey: 'obj' }, + { apiMethod: 'objectHead', httpMethod: 'HEAD', url: '/obj', query: {}, objectKey: 'obj' }, + { apiMethod: 'objectDelete', httpMethod: 'DELETE', url: '/obj', query: {}, objectKey: 'obj' }, + { + apiMethod: 'objectGetLegalHold', + httpMethod: 'GET', + url: '/obj?legal-hold', + query: { 'legal-hold': '' }, + objectKey: 'obj', + }, + { + apiMethod: 'objectGetAttributes', + httpMethod: 'GET', + url: '/obj?attributes', + query: { attributes: '' }, + objectKey: 'obj', + }, + { apiMethod: 'listMultipartUploads', httpMethod: 'GET', url: '/?uploads', query: { uploads: '' } }, ]; const bucketName = 'corserrorheaderstest'; @@ -95,24 +82,29 @@ function buildRequest(spec) { // DummyRequest is an http.IncomingMessage stream that emits 'end' // synchronously. We need that because callApiMethod's waterfall // waits for the request body on non-objectPut paths. - return new DummyRequest({ - bucketName, - objectKey: spec.objectKey, - headers: { - host: `${bucketName}.s3.amazonaws.com`, - origin, + return new DummyRequest( + { + bucketName, + objectKey: spec.objectKey, + headers: { + host: `${bucketName}.s3.amazonaws.com`, + origin, + }, + url: spec.url, + query: spec.query, + method: spec.httpMethod, }, - url: spec.url, - query: spec.query, - method: spec.httpMethod, - }, Buffer.alloc(0)); + Buffer.alloc(0), + ); } function buildResponseSpy(sandbox) { const headers = {}; return { headers, - setHeader: sandbox.spy((k, v) => { headers[k.toLowerCase()] = v; }), + setHeader: sandbox.spy((k, v) => { + headers[k.toLowerCase()] = v; + }), getHeader: k => headers[k.toLowerCase()], }; } @@ -126,7 +118,9 @@ function buildLog(sandbox) { warn: sandbox.stub(), error: sandbox.stub(), fatal: sandbox.stub(), - end() { return this; }, + end() { + return this; + }, }; } @@ -146,75 +140,65 @@ describe('CORS headers on 403 auth failures (api.callApiMethod)', () => { afterEach(() => sandbox.restore()); endpoints.forEach(spec => { - it(`attaches CORS headers to 403 response for ${spec.apiMethod}`, - done => { - const request = buildRequest(spec); - const response = buildResponseSpy(sandbox); - const log = buildLog(sandbox); - - api.callApiMethod(spec.apiMethod, request, response, log, - err => { - assert(err, 'expected an error'); - assert(err.is && err.is.AccessDenied, - `expected AccessDenied, got ${err.code}`); - // Either the callback surfaces CORS headers in one of - // its trailing args OR they have been set directly on - // the HTTP response. We assert on the response since - // that is what the HTTP transport ultimately sends. - const allowOrigin = response.getHeader( - 'access-control-allow-origin'); - assert(allowOrigin, - 'access-control-allow-origin missing from 403 ' - + `response for ${spec.apiMethod}`); - assert(response.getHeader( - 'access-control-allow-methods'), - 'access-control-allow-methods missing'); - done(); - }); - }); - }); - - it('does not attach CORS headers when Origin header is absent', - done => { - const request = buildRequest({ - apiMethod: 'bucketGet', httpMethod: 'GET', - url: '/', query: {}, - }); - delete request.headers.origin; + it(`attaches CORS headers to 403 response for ${spec.apiMethod}`, done => { + const request = buildRequest(spec); const response = buildResponseSpy(sandbox); const log = buildLog(sandbox); - api.callApiMethod('bucketGet', request, response, log, err => { - assert(err && err.is.AccessDenied); - assert.strictEqual( - response.getHeader('access-control-allow-origin'), - undefined); + api.callApiMethod(spec.apiMethod, request, response, log, err => { + assert(err, 'expected an error'); + assert(err.is && err.is.AccessDenied, `expected AccessDenied, got ${err.code}`); + // Either the callback surfaces CORS headers in one of + // its trailing args OR they have been set directly on + // the HTTP response. We assert on the response since + // that is what the HTTP transport ultimately sends. + const allowOrigin = response.getHeader('access-control-allow-origin'); + assert(allowOrigin, 'access-control-allow-origin missing from 403 ' + `response for ${spec.apiMethod}`); + assert(response.getHeader('access-control-allow-methods'), 'access-control-allow-methods missing'); done(); }); }); + }); - it('does not attach CORS headers when origin does not match any rule', - done => { - const request = buildRequest({ - apiMethod: 'bucketGet', httpMethod: 'GET', - url: '/', query: {}, - }); - request.headers.origin = 'http://not-allowed.test'; - // The bucket's CORS config allows any method from foo.test - // plus GET from *. Use PUT from a different origin so neither - // condition matches. - request.method = 'PUT'; - const response = buildResponseSpy(sandbox); - const log = buildLog(sandbox); + it('does not attach CORS headers when Origin header is absent', done => { + const request = buildRequest({ + apiMethod: 'bucketGet', + httpMethod: 'GET', + url: '/', + query: {}, + }); + delete request.headers.origin; + const response = buildResponseSpy(sandbox); + const log = buildLog(sandbox); - api.callApiMethod('bucketGet', request, response, log, err => { - assert(err && err.is.AccessDenied); - assert.strictEqual( - response.getHeader('access-control-allow-origin'), - undefined); - done(); - }); + api.callApiMethod('bucketGet', request, response, log, err => { + assert(err && err.is.AccessDenied); + assert.strictEqual(response.getHeader('access-control-allow-origin'), undefined); + done(); }); + }); + + it('does not attach CORS headers when origin does not match any rule', done => { + const request = buildRequest({ + apiMethod: 'bucketGet', + httpMethod: 'GET', + url: '/', + query: {}, + }); + request.headers.origin = 'http://not-allowed.test'; + // The bucket's CORS config allows any method from foo.test + // plus GET from *. Use PUT from a different origin so neither + // condition matches. + request.method = 'PUT'; + const response = buildResponseSpy(sandbox); + const log = buildLog(sandbox); + + api.callApiMethod('bucketGet', request, response, log, err => { + assert(err && err.is.AccessDenied); + assert.strictEqual(response.getHeader('access-control-allow-origin'), undefined); + done(); + }); + }); }); describe('CORS headers on 403 via handler (fast path)', () => { @@ -228,67 +212,63 @@ describe('CORS headers on 403 via handler (fast path)', () => { // denies at its own ACL check (bucket is owned by accessKey1). const otherAuth = makeAuthInfo('accessKey2'); const authServer = { - doAuth: sandbox.stub().callsArgWith(2, null, otherAuth, - [{ isAllowed: true, isImplicit: false }], null, {}), + doAuth: sandbox.stub().callsArgWith(2, null, otherAuth, [{ isAllowed: true, isImplicit: false }], null, {}), }; sandbox.stub(auth, 'server').value(authServer); }); afterEach(() => sandbox.restore()); - it('forwards handler-provided corsHeaders without setting headers ' - + 'on the response directly', done => { + it('forwards handler-provided corsHeaders without setting headers ' + 'on the response directly', done => { const request = buildRequest({ - apiMethod: 'bucketGet', httpMethod: 'GET', - url: '/', query: {}, + apiMethod: 'bucketGet', + httpMethod: 'GET', + url: '/', + query: {}, }); const response = buildResponseSpy(sandbox); const log = buildLog(sandbox); - api.callApiMethod('bucketGet', request, response, log, - (err, xml, corsHeaders) => { - assert(err, 'expected an error'); - assert(err.is && err.is.AccessDenied, - `expected AccessDenied, got ${err.code}`); - assert(corsHeaders, - 'handler should have supplied corsHeaders'); - assert.strictEqual( - corsHeaders['access-control-allow-origin'], origin); - // Fast path: wrapper forwards corsHeaders via the callback - // instead of setting them on the response directly. - assert.strictEqual( - response.getHeader('access-control-allow-origin'), - undefined); - done(); - }); + api.callApiMethod('bucketGet', request, response, log, (err, xml, corsHeaders) => { + assert(err, 'expected an error'); + assert(err.is && err.is.AccessDenied, `expected AccessDenied, got ${err.code}`); + assert(corsHeaders, 'handler should have supplied corsHeaders'); + assert.strictEqual(corsHeaders['access-control-allow-origin'], origin); + // Fast path: wrapper forwards corsHeaders via the callback + // instead of setting them on the response directly. + assert.strictEqual(response.getHeader('access-control-allow-origin'), undefined); + done(); + }); }); - it('makes at most 2 metadata.getBucket calls on the error path', - done => { - const getBucketSpy = sandbox.spy(metadata, 'getBucket'); - const request = buildRequest({ - apiMethod: 'bucketHead', httpMethod: 'HEAD', - url: '/', query: {}, - }); - // Origin that matches no CORS rule -> handler emits empty - // corsHeaders -> fast path misses -> wrapper falls back to a - // second getBucket. The handler's own call (1) + the wrapper - // fallback (1) is the documented ceiling - see the comment - // on wrapCallbackWithErrorCorsHeaders in lib/api/api.js. Use - // <= so this remains a future-proof ceiling: optimizations - // that reduce the count are welcome. - request.headers.origin = 'http://not-allowed.test'; - const response = buildResponseSpy(sandbox); - const log = buildLog(sandbox); + it('makes at most 2 metadata.getBucket calls on the error path', done => { + const getBucketSpy = sandbox.spy(metadata, 'getBucket'); + const request = buildRequest({ + apiMethod: 'bucketHead', + httpMethod: 'HEAD', + url: '/', + query: {}, + }); + // Origin that matches no CORS rule -> handler emits empty + // corsHeaders -> fast path misses -> wrapper falls back to a + // second getBucket. The handler's own call (1) + the wrapper + // fallback (1) is the documented ceiling - see the comment + // on wrapCallbackWithErrorCorsHeaders in lib/api/api.js. Use + // <= so this remains a future-proof ceiling: optimizations + // that reduce the count are welcome. + request.headers.origin = 'http://not-allowed.test'; + const response = buildResponseSpy(sandbox); + const log = buildLog(sandbox); - api.callApiMethod('bucketHead', request, response, log, err => { - assert(err && err.is.AccessDenied); - assert(getBucketSpy.callCount <= 2, - 'expected at most 2 metadata.getBucket calls, got ' - + `${getBucketSpy.callCount}`); - done(); - }); + api.callApiMethod('bucketHead', request, response, log, err => { + assert(err && err.is.AccessDenied); + assert( + getBucketSpy.callCount <= 2, + 'expected at most 2 metadata.getBucket calls, got ' + `${getBucketSpy.callCount}`, + ); + done(); }); + }); }); describe('CORS headers on copy operations', () => { @@ -331,31 +311,40 @@ describe('CORS headers on copy operations', () => { allowedMethods: ['PUT'], allowedOrigins: [reqOrigin], }); - async.series([ - cb => bucketPut(authInfo, destPutReq, log, cb), - cb => bucketPut(authInfo, srcPutReq, log, cb), - cb => bucketPutCors(authInfo, - destCors.createBucketCorsRequest('PUT', destBucket), log, cb), - cb => bucketPutCors(authInfo, - srcCors.createBucketCorsRequest('PUT', srcBucket), log, cb), - ], done); + async.series( + [ + cb => bucketPut(authInfo, destPutReq, log, cb), + cb => bucketPut(authInfo, srcPutReq, log, cb), + cb => bucketPutCors(authInfo, destCors.createBucketCorsRequest('PUT', destBucket), log, cb), + cb => bucketPutCors(authInfo, srcCors.createBucketCorsRequest('PUT', srcBucket), log, cb), + ], + done, + ); }); beforeEach(() => { sandbox = sinon.createSandbox(); const authServer = { - doAuth: sandbox.stub().callsArgWith(2, null, authInfo, - [{ isAllowed: true, isImplicit: false }, - { isAllowed: true, isImplicit: false }], null, {}), + doAuth: sandbox.stub().callsArgWith( + 2, + null, + authInfo, + [ + { isAllowed: true, isImplicit: false }, + { isAllowed: true, isImplicit: false }, + ], + null, + {}, + ), }; sandbox.stub(auth, 'server').value(authServer); }); afterEach(() => sandbox.restore()); - it('does not leak source-bucket CORS headers on objectCopy errors', - done => { - const request = new DummyRequest({ + it('does not leak source-bucket CORS headers on objectCopy errors', done => { + const request = new DummyRequest( + { bucketName: destBucket, objectKey: 'destkey', headers: { @@ -366,23 +355,26 @@ describe('CORS headers on copy operations', () => { url: `/${destBucket}/destkey`, query: {}, method: 'PUT', - }, Buffer.alloc(0)); - const response = buildResponseSpy(sandbox); - const log = buildLog(sandbox); + }, + Buffer.alloc(0), + ); + const response = buildResponseSpy(sandbox); + const log = buildLog(sandbox); - api.callApiMethod('objectCopy', request, response, log, err => { - assert(err, 'expected an error'); - // Dest does not allow PUT from reqOrigin, so no CORS - // headers should be set. If the wrapper used the source - // bucket (which DOES allow PUT from reqOrigin) we would - // see access-control-allow-origin: http://foo.test here. - assert.strictEqual( - response.getHeader('access-control-allow-origin'), - undefined, - 'wrapper must not apply source-bucket CORS headers'); - done(); - }); + api.callApiMethod('objectCopy', request, response, log, err => { + assert(err, 'expected an error'); + // Dest does not allow PUT from reqOrigin, so no CORS + // headers should be set. If the wrapper used the source + // bucket (which DOES allow PUT from reqOrigin) we would + // see access-control-allow-origin: http://foo.test here. + assert.strictEqual( + response.getHeader('access-control-allow-origin'), + undefined, + 'wrapper must not apply source-bucket CORS headers', + ); + done(); }); + }); }); describe('CORS headers on 200 successful responses (per-handler)', () => { @@ -403,10 +395,8 @@ describe('CORS headers on 200 successful responses (per-handler)', () => { }; bucketGet(authInfo, request, log, (err, xml, corsHeaders) => { assert.ifError(err); - assert(corsHeaders, - 'expected corsHeaders to be set on successful bucketGet'); - assert(corsHeaders['access-control-allow-origin'], - 'expected access-control-allow-origin on 200'); + assert(corsHeaders, 'expected corsHeaders to be set on successful bucketGet'); + assert(corsHeaders['access-control-allow-origin'], 'expected access-control-allow-origin on 200'); done(); }); }); @@ -426,9 +416,10 @@ describe('CORS headers on 200 successful responses (per-handler)', () => { }; bucketGetCors(authInfo, request, log, (err, xml, corsHeaders) => { assert.ifError(err); - assert(corsHeaders - && corsHeaders['access-control-allow-origin'], - 'expected access-control-allow-origin on 200'); + assert( + corsHeaders && corsHeaders['access-control-allow-origin'], + 'expected access-control-allow-origin on 200', + ); done(); }); }); diff --git a/tests/unit/api/createAndStoreObject.js b/tests/unit/api/createAndStoreObject.js index 1c28cce4f1..4f6f8a58d9 100644 --- a/tests/unit/api/createAndStoreObject.js +++ b/tests/unit/api/createAndStoreObject.js @@ -23,15 +23,16 @@ const canonicalID = authInfo.getCanonicalID(); const bucketName = 'test-bucket'; const objectKey = 'test-object'; -const getObjectMDAsync = (bucket, key, params = {}) => new Promise((resolve, reject) => { - metadata.getObjectMD(bucket, key, params, log, (err, data) => { - if (err) { - reject(err); - } else { - resolve(data); - } +const getObjectMDAsync = (bucket, key, params = {}) => + new Promise((resolve, reject) => { + metadata.getObjectMD(bucket, key, params, log, (err, data) => { + if (err) { + reject(err); + } else { + resolve(data); + } + }); }); -}); describe('createAndStoreObject', () => { let testBucket; @@ -64,17 +65,32 @@ describe('createAndStoreObject', () => { describe('Regular object creation', () => { it('should create object successfully', async () => { - const request = new DummyRequest({ + const request = new DummyRequest( + { + bucketName, + namespace: 'default', + objectKey, + headers: { 'content-type': 'text/plain' }, + url: `/${bucketName}/${objectKey}`, + }, + Buffer.from('test data', 'utf8'), + ); + + const result = await createAndStoreObject( bucketName, - namespace: 'default', + testBucket, objectKey, - headers: { 'content-type': 'text/plain' }, - url: `/${bucketName}/${objectKey}`, - }, Buffer.from('test data', 'utf8')); - - const result = await createAndStoreObject(bucketName, testBucket, objectKey, null, - authInfo, canonicalID, null, request, false, null, - ['overhead'], log, 's3:ObjectCreated:Put'); + null, + authInfo, + canonicalID, + null, + request, + false, + null, + ['overhead'], + log, + 's3:ObjectCreated:Put', + ); assert(result.contentMD5); @@ -83,18 +99,33 @@ describe('createAndStoreObject', () => { }); it('should handle zero-byte object', async () => { - const request = new DummyRequest({ + const request = new DummyRequest( + { + bucketName, + namespace: 'default', + objectKey, + headers: { 'content-type': 'text/plain' }, + parsedContentLength: 0, + url: `/${bucketName}/${objectKey}`, + }, + '', + ); + + const result = await createAndStoreObject( bucketName, - namespace: 'default', + testBucket, objectKey, - headers: { 'content-type': 'text/plain' }, - parsedContentLength: 0, - url: `/${bucketName}/${objectKey}`, - }, ''); - - const result = await createAndStoreObject(bucketName, testBucket, objectKey, null, - authInfo, canonicalID, null, request, false, null, - ['overhead'], log, 's3:ObjectCreated:Put'); + null, + authInfo, + canonicalID, + null, + request, + false, + null, + ['overhead'], + log, + 's3:ObjectCreated:Put', + ); assert(result.contentMD5); }); @@ -103,17 +134,32 @@ describe('createAndStoreObject', () => { const authInfo2 = makeAuthInfo('accessKey2'); sinon.spy(metadata, 'putObjectMD'); - const request = new DummyRequest({ + const request = new DummyRequest( + { + bucketName, + namespace: 'default', + objectKey, + headers: {}, + url: `/${bucketName}/${objectKey}`, + }, + Buffer.from('test', 'utf8'), + ); + + await createAndStoreObject( bucketName, - namespace: 'default', + testBucket, objectKey, - headers: {}, - url: `/${bucketName}/${objectKey}`, - }, Buffer.from('test', 'utf8')); - - await createAndStoreObject(bucketName, testBucket, objectKey, null, - authInfo2, authInfo2.getCanonicalID(), null, request, false, null, - ['overhead'], log, 's3:ObjectCreated:Put'); + null, + authInfo2, + authInfo2.getCanonicalID(), + null, + request, + false, + null, + ['overhead'], + log, + 's3:ObjectCreated:Put', + ); const storedObjMD = getStoredObjectData(); assert.strictEqual(storedObjMD.bucketOwnerId, canonicalID); @@ -130,9 +176,21 @@ describe('createAndStoreObject', () => { url: `/${bucketName}/${objectKey}`, }); - await createAndStoreObject(bucketName, testBucket, objectKey, null, - authInfo, canonicalID, null, request, true, null, - ['overhead'], log, 's3:ObjectRemoved:DeleteMarkerCreated'); + await createAndStoreObject( + bucketName, + testBucket, + objectKey, + null, + authInfo, + canonicalID, + null, + request, + true, + null, + ['overhead'], + log, + 's3:ObjectRemoved:DeleteMarkerCreated', + ); assert.deepStrictEqual(ds, []); @@ -146,24 +204,39 @@ describe('createAndStoreObject', () => { const archivedObjMD = { 'content-md5': 'abc123', 'content-length': 100, - 'archive': { - 'archiveInfo': { 'archiveID': 'archive-123' }, + archive: { + archiveInfo: { archiveID: 'archive-123' }, }, }; sinon.spy(metadata, 'putObjectMD'); - const request = new DummyRequest({ + const request = new DummyRequest( + { + bucketName, + namespace: 'default', + objectKey, + headers: {}, + url: `/${bucketName}/${objectKey}`, + }, + Buffer.from('new data', 'utf8'), + ); + + await createAndStoreObject( bucketName, - namespace: 'default', + testBucket, objectKey, - headers: {}, - url: `/${bucketName}/${objectKey}`, - }, Buffer.from('new data', 'utf8')); - - await createAndStoreObject(bucketName, testBucket, objectKey, archivedObjMD, - authInfo, canonicalID, null, request, false, null, - ['overhead'], log, 's3:ObjectCreated:Put'); + archivedObjMD, + authInfo, + canonicalID, + null, + request, + false, + null, + ['overhead'], + log, + 's3:ObjectCreated:Put', + ); const options = getStoredOptions(); assert.strictEqual(options.needOplogUpdate, true); @@ -174,26 +247,41 @@ describe('createAndStoreObject', () => { const archivedObjMD = { 'content-md5': 'abc123', 'content-length': 100, - 'versionId': 'v1', - 'archive': { - 'archiveInfo': { 'archiveID': 'archive-123' }, + versionId: 'v1', + archive: { + archiveInfo: { archiveID: 'archive-123' }, }, }; sinon.stub(testBucket, 'isVersioningEnabled').returns(true); sinon.spy(metadata, 'putObjectMD'); - const request = new DummyRequest({ + const request = new DummyRequest( + { + bucketName, + namespace: 'default', + objectKey, + headers: {}, + url: `/${bucketName}/${objectKey}`, + }, + Buffer.from('new data', 'utf8'), + ); + + await createAndStoreObject( bucketName, - namespace: 'default', + testBucket, objectKey, - headers: {}, - url: `/${bucketName}/${objectKey}`, - }, Buffer.from('new data', 'utf8')); - - await createAndStoreObject(bucketName, testBucket, objectKey, archivedObjMD, - authInfo, canonicalID, null, request, false, null, - ['overhead'], log, 's3:ObjectCreated:Put'); + archivedObjMD, + authInfo, + canonicalID, + null, + request, + false, + null, + ['overhead'], + log, + 's3:ObjectCreated:Put', + ); const options = getStoredOptions(); assert.strictEqual(options.needOplogUpdate, undefined); @@ -212,17 +300,32 @@ describe('createAndStoreObject', () => { sinon.stub(testBucket, 'isVersioningEnabled').returns(false); sinon.spy(metadata, 'putObjectMD'); - const request = new DummyRequest({ + const request = new DummyRequest( + { + bucketName, + namespace: 'default', + objectKey, + headers: {}, + url: `/${bucketName}/${objectKey}`, + }, + Buffer.from('new data', 'utf8'), + ); + + await createAndStoreObject( bucketName, - namespace: 'default', + testBucket, objectKey, - headers: {}, - url: `/${bucketName}/${objectKey}`, - }, Buffer.from('new data', 'utf8')); - - await createAndStoreObject(bucketName, testBucket, objectKey, archivedObjMD, - authInfo, canonicalID, null, request, false, null, - ['overhead'], log, 's3:ObjectCreated:Put'); + archivedObjMD, + authInfo, + canonicalID, + null, + request, + false, + null, + ['overhead'], + log, + 's3:ObjectCreated:Put', + ); const options = getStoredOptions(); assert.strictEqual(options.needOplogUpdate, true); @@ -239,17 +342,32 @@ describe('createAndStoreObject', () => { }; sinon.spy(metadata, 'putObjectMD'); - const request = new DummyRequest({ + const request = new DummyRequest( + { + bucketName, + namespace: 'default', + objectKey, + headers: {}, + url: `/${bucketName}/${objectKey}`, + }, + Buffer.from('new data', 'utf8'), + ); + + await createAndStoreObject( bucketName, - namespace: 'default', + testBucket, objectKey, - headers: {}, - url: `/${bucketName}/${objectKey}`, - }, Buffer.from('new data', 'utf8')); - - await createAndStoreObject(bucketName, testBucket, objectKey, archivedObjMD, - authInfo, canonicalID, null, request, false, null, - ['overhead'], log, 's3:ObjectCreated:Put'); + archivedObjMD, + authInfo, + canonicalID, + null, + request, + false, + null, + ['overhead'], + log, + 's3:ObjectCreated:Put', + ); const options = getStoredOptions(); assert.strictEqual(options.needOplogUpdate, undefined); @@ -261,34 +379,49 @@ describe('createAndStoreObject', () => { it('should restore object with x-scal-s3-version-id header', async () => { const now = Date.now(); const archivedObjMD = { - 'key': objectKey, - 'versionId': 'v123', + key: objectKey, + versionId: 'v123', 'content-md5': 'original-hash', 'content-length': 100, 'x-amz-storage-class': 'cold-location', - 'dataStoreName': 'cold-location', + dataStoreName: 'cold-location', 'x-amz-meta-custom': 'preserved-value', - 'tags': { 'tagkey': 'tagvalue' }, - 'archive': { - 'archiveInfo': { 'archiveID': 'archive-123' }, - 'restoreRequestedAt': new Date(now).toISOString(), - 'restoreRequestedDays': 7, + tags: { tagkey: 'tagvalue' }, + archive: { + archiveInfo: { archiveID: 'archive-123' }, + restoreRequestedAt: new Date(now).toISOString(), + restoreRequestedDays: 7, }, }; sinon.spy(metadata, 'putObjectMD'); - const request = new DummyRequest({ + const request = new DummyRequest( + { + bucketName, + namespace: 'default', + objectKey, + headers: { 'x-scal-s3-version-id': 'v123' }, + url: `/${bucketName}/${objectKey}`, + }, + Buffer.from('restored data', 'utf8'), + ); + + await createAndStoreObject( bucketName, - namespace: 'default', + testBucket, objectKey, - headers: { 'x-scal-s3-version-id': 'v123' }, - url: `/${bucketName}/${objectKey}`, - }, Buffer.from('restored data', 'utf8')); - - await createAndStoreObject(bucketName, testBucket, objectKey, archivedObjMD, - authInfo, canonicalID, null, request, false, null, - ['overhead'], log, 's3:ObjectCreated:Put'); + archivedObjMD, + authInfo, + canonicalID, + null, + request, + false, + null, + ['overhead'], + log, + 's3:ObjectCreated:Put', + ); const storedObjMD = getStoredObjectData(); const options = getStoredOptions(); @@ -296,7 +429,7 @@ describe('createAndStoreObject', () => { assert(storedObjMD.archive.restoreWillExpireAt, 'restoreWillExpireAt should be set'); assert.strictEqual(storedObjMD.archive.restoreRequestedDays, 7); assert.strictEqual(storedObjMD['x-amz-meta-custom'], 'preserved-value'); - assert.deepStrictEqual(storedObjMD.tags, { 'tagkey': 'tagvalue' }); + assert.deepStrictEqual(storedObjMD.tags, { tagkey: 'tagvalue' }); assert.strictEqual(storedObjMD.originOp, 's3:ObjectRestore:Completed'); assert.strictEqual(options.needOplogUpdate, undefined); assert.strictEqual(options.originOp, undefined); @@ -304,36 +437,49 @@ describe('createAndStoreObject', () => { it('should preserve original etag for MPU restoration with different part count', async () => { const archivedObjMD = { - 'versionId': 'v123', + versionId: 'v123', 'content-md5': 'original-abc123-5', // Original had 5 parts - 'archive': { - 'archiveInfo': { 'archiveID': 'archive-123' }, - 'restoreRequestedAt': new Date().toISOString(), - 'restoreRequestedDays': 7, + archive: { + archiveInfo: { archiveID: 'archive-123' }, + restoreRequestedAt: new Date().toISOString(), + restoreRequestedDays: 7, }, }; sinon.spy(metadata, 'putObjectMD'); - const request = new DummyRequest({ + const request = new DummyRequest( + { + bucketName, + namespace: 'default', + objectKey, + headers: { 'x-scal-s3-version-id': 'v123' }, + url: `/${bucketName}/${objectKey}`, + calculatedHash: 'restored-def456-3', // Restored with 3 parts + }, + Buffer.from('restored data', 'utf8'), + ); + + await createAndStoreObject( bucketName, - namespace: 'default', + testBucket, objectKey, - headers: { 'x-scal-s3-version-id': 'v123' }, - url: `/${bucketName}/${objectKey}`, - calculatedHash: 'restored-def456-3', // Restored with 3 parts - }, Buffer.from('restored data', 'utf8')); - - await createAndStoreObject(bucketName, testBucket, objectKey, archivedObjMD, - authInfo, canonicalID, null, request, false, null, - ['overhead'], log, 's3:ObjectCreated:Put'); + archivedObjMD, + authInfo, + canonicalID, + null, + request, + false, + null, + ['overhead'], + log, + 's3:ObjectCreated:Put', + ); const storedObjMD = getStoredObjectData(); - assert.strictEqual(storedObjMD['content-md5'], 'original-abc123-5', - 'Original etag should be preserved'); + assert.strictEqual(storedObjMD['content-md5'], 'original-abc123-5', 'Original etag should be preserved'); assert(storedObjMD['x-amz-restore']['content-md5']); - assert.notStrictEqual(storedObjMD['x-amz-restore']['content-md5'], - storedObjMD['content-md5']); + assert.notStrictEqual(storedObjMD['x-amz-restore']['content-md5'], storedObjMD['content-md5']); }); it('should preserve replication info during restoration', async () => { @@ -343,156 +489,237 @@ describe('createAndStoreObject', () => { }; const archivedObjMD = { - 'versionId': 'v123', + versionId: 'v123', replicationInfo, - 'archive': { - 'archiveInfo': { 'archiveID': 'archive-123' }, - 'restoreRequestedAt': new Date().toISOString(), - 'restoreRequestedDays': 7, + archive: { + archiveInfo: { archiveID: 'archive-123' }, + restoreRequestedAt: new Date().toISOString(), + restoreRequestedDays: 7, }, }; sinon.spy(metadata, 'putObjectMD'); - const request = new DummyRequest({ + const request = new DummyRequest( + { + bucketName, + namespace: 'default', + objectKey, + headers: { 'x-scal-s3-version-id': 'v123' }, + url: `/${bucketName}/${objectKey}`, + }, + Buffer.from('restored', 'utf8'), + ); + + await createAndStoreObject( bucketName, - namespace: 'default', + testBucket, objectKey, - headers: { 'x-scal-s3-version-id': 'v123' }, - url: `/${bucketName}/${objectKey}`, - }, Buffer.from('restored', 'utf8')); - - await createAndStoreObject(bucketName, testBucket, objectKey, archivedObjMD, - authInfo, canonicalID, null, request, false, null, - ['overhead'], log, 's3:ObjectCreated:Put'); + archivedObjMD, + authInfo, + canonicalID, + null, + request, + false, + null, + ['overhead'], + log, + 's3:ObjectCreated:Put', + ); const storedObjMD = getStoredObjectData(); - assert.strictEqual(storedObjMD.replicationInfo.status, replicationInfo.status, - 'Replication status should be preserved'); - assert.deepStrictEqual(storedObjMD.replicationInfo.backends, replicationInfo.backends, - 'Replication backends should be preserved'); + assert.strictEqual( + storedObjMD.replicationInfo.status, + replicationInfo.status, + 'Replication status should be preserved', + ); + assert.deepStrictEqual( + storedObjMD.replicationInfo.backends, + replicationInfo.backends, + 'Replication backends should be preserved', + ); }); it('should preserve legal hold during restoration', async () => { const archivedObjMD = { - 'versionId': 'v123', - 'legalHold': true, - 'archive': { - 'archiveInfo': { 'archiveID': 'archive-123' }, - 'restoreRequestedAt': new Date().toISOString(), - 'restoreRequestedDays': 7, + versionId: 'v123', + legalHold: true, + archive: { + archiveInfo: { archiveID: 'archive-123' }, + restoreRequestedAt: new Date().toISOString(), + restoreRequestedDays: 7, }, }; sinon.spy(metadata, 'putObjectMD'); - const request = new DummyRequest({ + const request = new DummyRequest( + { + bucketName, + namespace: 'default', + objectKey, + headers: { 'x-scal-s3-version-id': 'v123' }, + url: `/${bucketName}/${objectKey}`, + }, + Buffer.from('restored', 'utf8'), + ); + + await createAndStoreObject( bucketName, - namespace: 'default', + testBucket, objectKey, - headers: { 'x-scal-s3-version-id': 'v123' }, - url: `/${bucketName}/${objectKey}`, - }, Buffer.from('restored', 'utf8')); - - await createAndStoreObject(bucketName, testBucket, objectKey, archivedObjMD, - authInfo, canonicalID, null, request, false, null, - ['overhead'], log, 's3:ObjectCreated:Put'); + archivedObjMD, + authInfo, + canonicalID, + null, + request, + false, + null, + ['overhead'], + log, + 's3:ObjectCreated:Put', + ); const storedObjMD = getStoredObjectData(); - assert.strictEqual(storedObjMD.legalHold, true, - 'Legal hold should be preserved'); + assert.strictEqual(storedObjMD.legalHold, true, 'Legal hold should be preserved'); }); it('should preserve ACLs during restoration', async () => { const acl = { - 'Canned': '', - 'FULL_CONTROL': ['canonical-id-1'], - 'READ': ['canonical-id-2'], + Canned: '', + FULL_CONTROL: ['canonical-id-1'], + READ: ['canonical-id-2'], }; const archivedObjMD = { - 'versionId': 'v123', + versionId: 'v123', acl, - 'archive': { - 'archiveInfo': { 'archiveID': 'archive-123' }, - 'restoreRequestedAt': new Date().toISOString(), - 'restoreRequestedDays': 7, + archive: { + archiveInfo: { archiveID: 'archive-123' }, + restoreRequestedAt: new Date().toISOString(), + restoreRequestedDays: 7, }, }; sinon.spy(metadata, 'putObjectMD'); - const request = new DummyRequest({ + const request = new DummyRequest( + { + bucketName, + namespace: 'default', + objectKey, + headers: { 'x-scal-s3-version-id': 'v123' }, + url: `/${bucketName}/${objectKey}`, + }, + Buffer.from('restored', 'utf8'), + ); + + await createAndStoreObject( bucketName, - namespace: 'default', + testBucket, objectKey, - headers: { 'x-scal-s3-version-id': 'v123' }, - url: `/${bucketName}/${objectKey}`, - }, Buffer.from('restored', 'utf8')); - - await createAndStoreObject(bucketName, testBucket, objectKey, archivedObjMD, - authInfo, canonicalID, null, request, false, null, - ['overhead'], log, 's3:ObjectCreated:Put'); + archivedObjMD, + authInfo, + canonicalID, + null, + request, + false, + null, + ['overhead'], + log, + 's3:ObjectCreated:Put', + ); const storedObjMD = getStoredObjectData(); - assert.deepStrictEqual(storedObjMD.acl, acl, - 'ACLs should be preserved'); + assert.deepStrictEqual(storedObjMD.acl, acl, 'ACLs should be preserved'); }); it('should not preserve x-amz-meta-scal-s3-restore-attempt metadata', async () => { const archivedObjMD = { - 'versionId': 'v123', + versionId: 'v123', 'x-amz-meta-custom': 'keep-this', 'x-amz-meta-scal-s3-restore-attempt': '3', - 'archive': { - 'archiveInfo': { 'archiveID': 'archive-123' }, - 'restoreRequestedAt': new Date().toISOString(), - 'restoreRequestedDays': 7, + archive: { + archiveInfo: { archiveID: 'archive-123' }, + restoreRequestedAt: new Date().toISOString(), + restoreRequestedDays: 7, }, }; sinon.spy(metadata, 'putObjectMD'); - const request = new DummyRequest({ + const request = new DummyRequest( + { + bucketName, + namespace: 'default', + objectKey, + headers: { 'x-scal-s3-version-id': 'v123' }, + url: `/${bucketName}/${objectKey}`, + }, + Buffer.from('restored', 'utf8'), + ); + + await createAndStoreObject( bucketName, - namespace: 'default', + testBucket, objectKey, - headers: { 'x-scal-s3-version-id': 'v123' }, - url: `/${bucketName}/${objectKey}`, - }, Buffer.from('restored', 'utf8')); - - await createAndStoreObject(bucketName, testBucket, objectKey, archivedObjMD, - authInfo, canonicalID, null, request, false, null, - ['overhead'], log, 's3:ObjectCreated:Put'); + archivedObjMD, + authInfo, + canonicalID, + null, + request, + false, + null, + ['overhead'], + log, + 's3:ObjectCreated:Put', + ); const storedObjMD = getStoredObjectData(); - assert.strictEqual(storedObjMD['x-amz-meta-custom'], 'keep-this', - 'Custom metadata should be preserved'); - assert.strictEqual(storedObjMD['x-amz-meta-scal-s3-restore-attempt'], undefined, - 'Restore attempt metadata should NOT be preserved'); + assert.strictEqual(storedObjMD['x-amz-meta-custom'], 'keep-this', 'Custom metadata should be preserved'); + assert.strictEqual( + storedObjMD['x-amz-meta-scal-s3-restore-attempt'], + undefined, + 'Restore attempt metadata should NOT be preserved', + ); }); }); describe('MPU scenarios', () => { it('should set oldReplayId when overwriting MPU object', async () => { const mpuObjMD = { - 'uploadId': 'mpu-upload-123', + uploadId: 'mpu-upload-123', 'content-md5': 'abc123', }; sinon.spy(metadata, 'putObjectMD'); - const request = new DummyRequest({ + const request = new DummyRequest( + { + bucketName, + namespace: 'default', + objectKey, + headers: {}, + url: `/${bucketName}/${objectKey}`, + }, + Buffer.from('new data', 'utf8'), + ); + + await createAndStoreObject( bucketName, - namespace: 'default', + testBucket, objectKey, - headers: {}, - url: `/${bucketName}/${objectKey}`, - }, Buffer.from('new data', 'utf8')); - - await createAndStoreObject(bucketName, testBucket, objectKey, mpuObjMD, - authInfo, canonicalID, null, request, false, null, - ['overhead'], log, 's3:ObjectCreated:Put'); + mpuObjMD, + authInfo, + canonicalID, + null, + request, + false, + null, + ['overhead'], + log, + 's3:ObjectCreated:Put', + ); const options = getStoredOptions(); assert.strictEqual(options.oldReplayId, 'mpu-upload-123'); @@ -508,17 +735,32 @@ describe('createAndStoreObject', () => { sinon.spy(metadata, 'putObjectMD'); - const request = new DummyRequest({ + const request = new DummyRequest( + { + bucketName, + namespace: 'default', + objectKey, + headers: {}, + url: `/${bucketName}/${objectKey}`, + }, + Buffer.from('new data', 'utf8'), + ); + + await createAndStoreObject( bucketName, - namespace: 'default', + testBucket, objectKey, - headers: {}, - url: `/${bucketName}/${objectKey}`, - }, Buffer.from('new data', 'utf8')); - - await createAndStoreObject(bucketName, testBucket, objectKey, existingObjMD, - authInfo, canonicalID, null, request, false, null, - ['overhead'], log, 's3:ObjectCreated:Put'); + existingObjMD, + authInfo, + canonicalID, + null, + request, + false, + null, + ['overhead'], + log, + 's3:ObjectCreated:Put', + ); const storedObjMD = getStoredObjectData(); assert.strictEqual(storedObjMD['creation-time'], '2024-01-01T00:00:00.000Z'); @@ -531,17 +773,32 @@ describe('createAndStoreObject', () => { sinon.spy(metadata, 'putObjectMD'); - const request = new DummyRequest({ + const request = new DummyRequest( + { + bucketName, + namespace: 'default', + objectKey, + headers: {}, + url: `/${bucketName}/${objectKey}`, + }, + Buffer.from('new data', 'utf8'), + ); + + await createAndStoreObject( bucketName, - namespace: 'default', + testBucket, objectKey, - headers: {}, - url: `/${bucketName}/${objectKey}`, - }, Buffer.from('new data', 'utf8')); - - await createAndStoreObject(bucketName, testBucket, objectKey, existingObjMD, - authInfo, canonicalID, null, request, false, null, - ['overhead'], log, 's3:ObjectCreated:Put'); + existingObjMD, + authInfo, + canonicalID, + null, + request, + false, + null, + ['overhead'], + log, + 's3:ObjectCreated:Put', + ); const storedObjMD = getStoredObjectData(); assert.strictEqual(storedObjMD['creation-time'], '2024-02-01T00:00:00.000Z'); @@ -562,17 +819,32 @@ describe('createAndStoreObject', () => { }, }; - const request = new DummyRequest({ + const request = new DummyRequest( + { + bucketName, + namespace: 'default', + objectKey, + headers: { 'x-scal-s3-version-id': putVersionId }, + url: `/${bucketName}/${objectKey}`, + }, + Buffer.from('restored', 'utf8'), + ); + + await createAndStoreObject( bucketName, - namespace: 'default', + testBucket, objectKey, - headers: { 'x-scal-s3-version-id': putVersionId }, - url: `/${bucketName}/${objectKey}`, - }, Buffer.from('restored', 'utf8')); - - await createAndStoreObject(bucketName, testBucket, objectKey, archivedObjMD, - authInfo, canonicalID, null, request, false, null, - ['overhead'], log, 's3:ObjectCreated:Put'); + archivedObjMD, + authInfo, + canonicalID, + null, + request, + false, + null, + ['overhead'], + log, + 's3:ObjectCreated:Put', + ); const storedObjMD = getStoredObjectData(); assert.strictEqual(storedObjMD['x-amz-meta-scal-version-id'], putVersionId); diff --git a/tests/unit/api/deleteMarker.js b/tests/unit/api/deleteMarker.js index a5610c057e..1425990094 100644 --- a/tests/unit/api/deleteMarker.js +++ b/tests/unit/api/deleteMarker.js @@ -48,10 +48,11 @@ function _createBucketPutVersioningReq(status) { query: { versioning: '' }, actionImplicitDenies: false, }; - const xml = '' + - `${status}` + - ''; + const xml = + '' + + `${status}` + + ''; request.post = xml; return request; } @@ -79,8 +80,7 @@ function _createMultiObjectDeleteRequest(numObjects) { } xml.push(''); request.post = xml.join(''); - request.headers['content-md5'] = crypto.createHash('md5') - .update(request.post, 'utf8').digest('base64'); + request.headers['content-md5'] = crypto.createHash('md5').update(request.post, 'utf8').digest('base64'); return request; } @@ -94,12 +94,7 @@ const expectedAcl = { READ_ACP: [], }; -const undefHeadersExpected = [ - 'cache-control', - 'content-disposition', - 'content-encoding', - 'expires', -]; +const undefHeadersExpected = ['cache-control', 'content-disposition', 'content-encoding', 'expires']; describe('delete marker creation', () => { beforeEach(done => { @@ -108,8 +103,7 @@ describe('delete marker creation', () => { if (err) { return done(err); } - return bucketPutVersioning(authInfo, enableVersioningRequest, - log, done); + return bucketPutVersioning(authInfo, enableVersioningRequest, log, done); }); }); @@ -119,46 +113,43 @@ describe('delete marker creation', () => { function _assertDeleteMarkerMd(deleteResultVersionId, isLatest, callback) { const options = { - versionId: isLatest ? undefined : - versionIdUtils.decode(deleteResultVersionId), + versionId: isLatest ? undefined : versionIdUtils.decode(deleteResultVersionId), }; - return metadata.getObjectMD(bucketName, objectName, options, log, - (err, deleteMarkerMD) => { - assert.strictEqual(err, null); - const mdVersionId = deleteMarkerMD.versionId; - assert.strictEqual(deleteMarkerMD.isDeleteMarker, true); - assert.strictEqual( - versionIdUtils.encode(mdVersionId), - deleteResultVersionId); - assert.strictEqual(deleteMarkerMD['content-length'], 0); - assert.strictEqual(deleteMarkerMD.location, null); - assert.deepStrictEqual(deleteMarkerMD.acl, expectedAcl); - undefHeadersExpected.forEach(header => { - assert.strictEqual(deleteMarkerMD[header], undefined); - }); - return callback(); + return metadata.getObjectMD(bucketName, objectName, options, log, (err, deleteMarkerMD) => { + assert.strictEqual(err, null); + const mdVersionId = deleteMarkerMD.versionId; + assert.strictEqual(deleteMarkerMD.isDeleteMarker, true); + assert.strictEqual(versionIdUtils.encode(mdVersionId), deleteResultVersionId); + assert.strictEqual(deleteMarkerMD['content-length'], 0); + assert.strictEqual(deleteMarkerMD.location, null); + assert.deepStrictEqual(deleteMarkerMD.acl, expectedAcl); + undefHeadersExpected.forEach(header => { + assert.strictEqual(deleteMarkerMD[header], undefined); }); + return callback(); + }); } - it('should create a delete marker if versioning enabled and deleting ' + - 'object without specifying version id', done => { - objectDelete(authInfo, testDeleteRequest, log, (err, delResHeaders) => { - if (err) { - return done(err); - } - assert.strictEqual(delResHeaders['x-amz-delete-marker'], true); - assert(delResHeaders['x-amz-version-id']); - return _assertDeleteMarkerMd(delResHeaders['x-amz-version-id'], - true, done); - }); - }); + it( + 'should create a delete marker if versioning enabled and deleting ' + 'object without specifying version id', + done => { + objectDelete(authInfo, testDeleteRequest, log, (err, delResHeaders) => { + if (err) { + return done(err); + } + assert.strictEqual(delResHeaders['x-amz-delete-marker'], true); + assert(delResHeaders['x-amz-version-id']); + return _assertDeleteMarkerMd(delResHeaders['x-amz-version-id'], true, done); + }); + }, + ); - it('multi-object delete should create delete markers if versioning ' + - 'enabled and items do not have version id specified', done => { - const testMultiObjectDeleteRequest = - _createMultiObjectDeleteRequest(3); - return multiObjectDelete(authInfo, testMultiObjectDeleteRequest, log, - (err, xml) => { + it( + 'multi-object delete should create delete markers if versioning ' + + 'enabled and items do not have version id specified', + done => { + const testMultiObjectDeleteRequest = _createMultiObjectDeleteRequest(3); + return multiObjectDelete(authInfo, testMultiObjectDeleteRequest, log, (err, xml) => { if (err) { return done(err); } @@ -167,14 +158,18 @@ describe('delete marker creation', () => { return done(err); } const results = parsedResult.DeleteResult.Deleted; - return async.forEach(results, (result, cb) => { - assert.strictEqual(result.Key[0], objectName); - assert.strictEqual(result.DeleteMarker[0], 'true'); - assert(result.DeleteMarkerVersionId[0]); - _assertDeleteMarkerMd(result.DeleteMarkerVersionId[0], - false, cb); - }, err => done(err)); + return async.forEach( + results, + (result, cb) => { + assert.strictEqual(result.Key[0], objectName); + assert.strictEqual(result.DeleteMarker[0], 'true'); + assert(result.DeleteMarkerVersionId[0]); + _assertDeleteMarkerMd(result.DeleteMarkerVersionId[0], false, cb); + }, + err => done(err), + ); }); }); - }); + }, + ); }); diff --git a/tests/unit/api/deletedFlagBucket.js b/tests/unit/api/deletedFlagBucket.js index 83d699f4e2..1d3198e795 100644 --- a/tests/unit/api/deletedFlagBucket.js +++ b/tests/unit/api/deletedFlagBucket.js @@ -14,18 +14,12 @@ const bucketPutWebsite = require('../../../lib/api/bucketPutWebsite'); const bucketDelete = require('../../../lib/api/bucketDelete'); const bucketDeleteCors = require('../../../lib/api/bucketDeleteCors'); const bucketDeleteWebsite = require('../../../lib/api/bucketDeleteWebsite'); -const completeMultipartUpload - = require('../../../lib/api/completeMultipartUpload'); +const completeMultipartUpload = require('../../../lib/api/completeMultipartUpload'); const { config } = require('../../../lib/Config'); const constants = require('../../../constants'); const DummyRequest = require('../DummyRequest'); -const initiateMultipartUpload - = require('../../../lib/api/initiateMultipartUpload'); -const { cleanup, - createAlteredRequest, - DummyRequestLogger, - makeAuthInfo } - = require('../helpers'); +const initiateMultipartUpload = require('../../../lib/api/initiateMultipartUpload'); +const { cleanup, createAlteredRequest, DummyRequestLogger, makeAuthInfo } = require('../helpers'); const listMultipartUploads = require('../../../lib/api/listMultipartUploads'); const listParts = require('../../../lib/api/listParts'); const metadata = require('../metadataswitch'); @@ -69,21 +63,16 @@ const serviceGetRequest = { const userBucketOwner = 'admin'; const creationDate = new Date().toJSON(); -const usersBucket = new BucketInfo(usersBucketName, - userBucketOwner, userBucketOwner, creationDate); - +const usersBucket = new BucketInfo(usersBucketName, userBucketOwner, userBucketOwner, creationDate); function checkBucketListing(authInfo, bucketName, expectedListingLength, done) { return serviceGet(authInfo, serviceGetRequest, log, (err, data) => { parseString(data, (err, result) => { if (expectedListingLength > 0) { - assert.strictEqual(result.ListAllMyBucketsResult - .Buckets[0].Bucket.length, expectedListingLength); - assert.strictEqual(result.ListAllMyBucketsResult - .Buckets[0].Bucket[0].Name[0], bucketName); + assert.strictEqual(result.ListAllMyBucketsResult.Buckets[0].Bucket.length, expectedListingLength); + assert.strictEqual(result.ListAllMyBucketsResult.Buckets[0].Bucket[0].Name[0], bucketName); } else { - assert.strictEqual(result.ListAllMyBucketsResult - .Buckets[0].length, 0); + assert.strictEqual(result.ListAllMyBucketsResult.Buckets[0].length, 0); } done(); }); @@ -105,15 +94,12 @@ function confirmDeleted(done) { }); } - describe('deleted flag bucket handling', () => { beforeEach(done => { cleanup(); - const bucketMD = new BucketInfo(bucketName, canonicalID, - authInfo.getAccountDisplayName(), creationDate); + const bucketMD = new BucketInfo(bucketName, canonicalID, authInfo.getAccountDisplayName(), creationDate); bucketMD.addDeletedFlag(); - bucketMD.setSpecificAcl(otherAccountAuthInfo.getCanonicalID(), - 'FULL_CONTROL'); + bucketMD.setSpecificAcl(otherAccountAuthInfo.getCanonicalID(), 'FULL_CONTROL'); bucketMD.setLocationConstraint(locationConstraint); metadata.createBucket(bucketName, bucketMD, log, () => { metadata.createBucket(usersBucketName, usersBucket, log, () => { @@ -122,77 +108,103 @@ describe('deleted flag bucket handling', () => { }); }); - it('putBucket request should recreate bucket with deleted flag if ' + - 'request is from same account that originally put', done => { - bucketPut(authInfo, baseTestRequest, log, err => { - assert.ifError(err); - metadata.getBucket(bucketName, log, (err, data) => { - assert.strictEqual(data._transient, false); - assert.strictEqual(data._deleted, false); - assert.strictEqual(data._owner, authInfo.getCanonicalID()); - return checkBucketListing(authInfo, bucketName, 1, done); + it( + 'putBucket request should recreate bucket with deleted flag if ' + + 'request is from same account that originally put', + done => { + bucketPut(authInfo, baseTestRequest, log, err => { + assert.ifError(err); + metadata.getBucket(bucketName, log, (err, data) => { + assert.strictEqual(data._transient, false); + assert.strictEqual(data._deleted, false); + assert.strictEqual(data._owner, authInfo.getCanonicalID()); + return checkBucketListing(authInfo, bucketName, 1, done); + }); }); - }); - }); + }, + ); - it('putBucket request should return error if ' + - 'different account sends put bucket request for bucket with ' + - 'deleted flag', done => { - bucketPut(otherAccountAuthInfo, baseTestRequest, log, err => { - assert.strictEqual(err.is.BucketAlreadyExists, true); - metadata.getBucket(bucketName, log, (err, data) => { - assert.strictEqual(data._transient, false); - assert.strictEqual(data._deleted, true); - assert.strictEqual(data._owner, authInfo.getCanonicalID()); - return checkBucketListing(otherAccountAuthInfo, - bucketName, 0, done); + it( + 'putBucket request should return error if ' + + 'different account sends put bucket request for bucket with ' + + 'deleted flag', + done => { + bucketPut(otherAccountAuthInfo, baseTestRequest, log, err => { + assert.strictEqual(err.is.BucketAlreadyExists, true); + metadata.getBucket(bucketName, log, (err, data) => { + assert.strictEqual(data._transient, false); + assert.strictEqual(data._deleted, true); + assert.strictEqual(data._owner, authInfo.getCanonicalID()); + return checkBucketListing(otherAccountAuthInfo, bucketName, 0, done); + }); }); - }); - }); + }, + ); - it('ACLs from new putBucket request should overwrite ACLs saved ' + - 'in metadata of bucket with deleted flag', done => { - const alteredRequest = createAlteredRequest({ - 'x-amz-acl': 'public-read' }, 'headers', - baseTestRequest, baseTestRequest.headers); - bucketPut(authInfo, alteredRequest, log, err => { - assert.ifError(err); - metadata.getBucket(bucketName, log, (err, data) => { - assert.strictEqual(data._transient, false); - assert.strictEqual(data._deleted, false); - assert.strictEqual(data._acl.Canned, 'public-read'); - assert.strictEqual(data._owner, authInfo.getCanonicalID()); - return checkBucketListing(authInfo, bucketName, 1, done); + it( + 'ACLs from new putBucket request should overwrite ACLs saved ' + 'in metadata of bucket with deleted flag', + done => { + const alteredRequest = createAlteredRequest( + { + 'x-amz-acl': 'public-read', + }, + 'headers', + baseTestRequest, + baseTestRequest.headers, + ); + bucketPut(authInfo, alteredRequest, log, err => { + assert.ifError(err); + metadata.getBucket(bucketName, log, (err, data) => { + assert.strictEqual(data._transient, false); + assert.strictEqual(data._deleted, false); + assert.strictEqual(data._acl.Canned, 'public-read'); + assert.strictEqual(data._owner, authInfo.getCanonicalID()); + return checkBucketListing(authInfo, bucketName, 1, done); + }); }); - }); - }); + }, + ); - it('putBucketACL request should recreate bucket with deleted flag if ' + - 'request is from same account that originally put', done => { - const putACLRequest = createAlteredRequest({ - 'x-amz-acl': 'public-read' }, 'headers', - baseTestRequest, baseTestRequest.headers); - putACLRequest.query = { acl: '' }; - bucketPutACL(authInfo, putACLRequest, log, err => { - assert.ifError(err); - metadata.getBucket(bucketName, log, (err, data) => { - assert.strictEqual(data._transient, false); - assert.strictEqual(data._acl.Canned, 'public-read'); - assert.strictEqual(data._owner, authInfo.getCanonicalID()); - return checkBucketListing(authInfo, bucketName, 1, done); + it( + 'putBucketACL request should recreate bucket with deleted flag if ' + + 'request is from same account that originally put', + done => { + const putACLRequest = createAlteredRequest( + { + 'x-amz-acl': 'public-read', + }, + 'headers', + baseTestRequest, + baseTestRequest.headers, + ); + putACLRequest.query = { acl: '' }; + bucketPutACL(authInfo, putACLRequest, log, err => { + assert.ifError(err); + metadata.getBucket(bucketName, log, (err, data) => { + assert.strictEqual(data._transient, false); + assert.strictEqual(data._acl.Canned, 'public-read'); + assert.strictEqual(data._owner, authInfo.getCanonicalID()); + return checkBucketListing(authInfo, bucketName, 1, done); + }); }); - }); - }); + }, + ); - it('putBucketACL request on bucket with deleted flag should return ' + - 'NoSuchBucket error if request is from another authorized account', + it( + 'putBucketACL request on bucket with deleted flag should return ' + + 'NoSuchBucket error if request is from another authorized account', // Do not want different account recreating a bucket that the bucket // owner wanted deleted even if the other account is authorized to // change the ACLs done => { - const putACLRequest = createAlteredRequest({ - 'x-amz-acl': 'public-read' }, 'headers', - baseTestRequest, baseTestRequest.headers); + const putACLRequest = createAlteredRequest( + { + 'x-amz-acl': 'public-read', + }, + 'headers', + baseTestRequest, + baseTestRequest.headers, + ); bucketPutACL(otherAccountAuthInfo, putACLRequest, log, err => { assert.strictEqual(err.is.NoSuchBucket, true); metadata.getBucket(bucketName, log, (err, data) => { @@ -203,14 +215,21 @@ describe('deleted flag bucket handling', () => { done(); }); }); - }); + }, + ); - it('putBucketACL request on bucket with deleted flag should return ' + - 'AccessDenied error if request is from unauthorized account', + it( + 'putBucketACL request on bucket with deleted flag should return ' + + 'AccessDenied error if request is from unauthorized account', done => { - const putACLRequest = createAlteredRequest({ - 'x-amz-acl': 'public-read' }, 'headers', - baseTestRequest, baseTestRequest.headers); + const putACLRequest = createAlteredRequest( + { + 'x-amz-acl': 'public-read', + }, + 'headers', + baseTestRequest, + baseTestRequest.headers, + ); const unauthorizedAccount = makeAuthInfo('keepMeOut'); bucketPutACL(unauthorizedAccount, putACLRequest, log, err => { assert.strictEqual(err.is.AccessDenied, true); @@ -222,7 +241,8 @@ describe('deleted flag bucket handling', () => { done(); }); }); - }); + }, + ); describe('objectPut on a bucket with deleted flag', () => { const objName = 'objectName'; @@ -232,10 +252,8 @@ describe('deleted flag bucket handling', () => { }); }); - it('objectPut request from account that originally created ' + - 'should recreate bucket', done => { - const setUpRequest = createAlteredRequest({}, 'headers', - baseTestRequest, baseTestRequest.headers); + it('objectPut request from account that originally created ' + 'should recreate bucket', done => { + const setUpRequest = createAlteredRequest({}, 'headers', baseTestRequest, baseTestRequest.headers); setUpRequest.objectKey = objName; const postBody = Buffer.from('I am a body', 'utf8'); const md5Hash = crypto.createHash('md5'); @@ -247,46 +265,43 @@ describe('deleted flag bucket handling', () => { assert.strictEqual(data._transient, false); assert.strictEqual(data._deleted, false); assert.strictEqual(data._owner, authInfo.getCanonicalID()); - metadata.getObjectMD(bucketName, objName, {}, log, - (err, obj) => { - assert.ifError(err); - assert.strictEqual(obj['content-md5'], etag); - return checkBucketListing(authInfo, - bucketName, 1, done); - }); + metadata.getObjectMD(bucketName, objName, {}, log, (err, obj) => { + assert.ifError(err); + assert.strictEqual(obj['content-md5'], etag); + return checkBucketListing(authInfo, bucketName, 1, done); + }); }); }); }); }); - it('should return NoSuchBucket error on an objectPut request from ' + - 'different account when there is a deleted flag', done => { - const setUpRequest = createAlteredRequest({}, 'headers', - baseTestRequest, baseTestRequest.headers); - setUpRequest.objectKey = 'objectName'; - const postBody = Buffer.from('I am a body', 'utf8'); - const putObjRequest = new DummyRequest(setUpRequest, postBody); - objectPut(otherAccountAuthInfo, putObjRequest, undefined, log, err => { - assert.strictEqual(err.is.NoSuchBucket, true); - done(); - }); - }); + it( + 'should return NoSuchBucket error on an objectPut request from ' + + 'different account when there is a deleted flag', + done => { + const setUpRequest = createAlteredRequest({}, 'headers', baseTestRequest, baseTestRequest.headers); + setUpRequest.objectKey = 'objectName'; + const postBody = Buffer.from('I am a body', 'utf8'); + const putObjRequest = new DummyRequest(setUpRequest, postBody); + objectPut(otherAccountAuthInfo, putObjRequest, undefined, log, err => { + assert.strictEqual(err.is.NoSuchBucket, true); + done(); + }); + }, + ); describe('initiateMultipartUpload on a bucket with deleted flag', () => { const objName = 'objectName'; after(done => { - metadata.deleteObjectMD(`${constants.mpuBucketPrefix}` + - `${bucketName}`, objName, {}, log, () => { - metadata.deleteBucket(`${constants.mpuBucketPrefix}` + - `${bucketName}`, log, () => { - done(); - }); + metadata.deleteObjectMD(`${constants.mpuBucketPrefix}` + `${bucketName}`, objName, {}, log, () => { + metadata.deleteBucket(`${constants.mpuBucketPrefix}` + `${bucketName}`, log, () => { + done(); }); + }); }); it('should recreate bucket with deleted flag', done => { - const initiateRequest = createAlteredRequest({}, 'headers', - baseTestRequest, baseTestRequest.headers); + const initiateRequest = createAlteredRequest({}, 'headers', baseTestRequest, baseTestRequest.headers); initiateRequest.objectKey = objName; initiateRequest.url = `/${objName}?uploads`; initiateMultipartUpload(authInfo, initiateRequest, log, err => { @@ -295,185 +310,196 @@ describe('deleted flag bucket handling', () => { assert.strictEqual(data._transient, false); assert.strictEqual(data._deleted, false); assert.strictEqual(data._owner, authInfo.getCanonicalID()); - metadata.listObject(`${constants.mpuBucketPrefix}` + - `${bucketName}`, + metadata.listObject( + `${constants.mpuBucketPrefix}` + `${bucketName}`, { prefix: `overview${constants.splitter}${objName}` }, - log, (err, results) => { + log, + (err, results) => { assert.ifError(err); assert.strictEqual(results.Contents.length, 1); done(); - }); + }, + ); }); }); }); }); - it('should return NoSuchBucket error on an initiateMultipartUpload ' + - 'request from different account when there is a deleted flag', done => { - const initiateRequest = createAlteredRequest({}, 'headers', - baseTestRequest, baseTestRequest.headers); - initiateRequest.objectKey = 'objectName'; - initiateMultipartUpload(otherAccountAuthInfo, initiateRequest, log, - err => { + it( + 'should return NoSuchBucket error on an initiateMultipartUpload ' + + 'request from different account when there is a deleted flag', + done => { + const initiateRequest = createAlteredRequest({}, 'headers', baseTestRequest, baseTestRequest.headers); + initiateRequest.objectKey = 'objectName'; + initiateMultipartUpload(otherAccountAuthInfo, initiateRequest, log, err => { assert.strictEqual(err.is.NoSuchBucket, true); done(); }); - }); + }, + ); - it('deleteBucket request should complete deletion ' + - 'of bucket with deleted flag', done => { + it('deleteBucket request should complete deletion ' + 'of bucket with deleted flag', done => { bucketDelete(authInfo, baseTestRequest, log, err => { assert.ifError(err); confirmDeleted(done); }); }); - it('deleteBucket request should return error if account not ' + - 'authorized', done => { - bucketDelete(otherAccountAuthInfo, baseTestRequest, - log, err => { - assert.strictEqual(err.is.AccessDenied, true); - done(); - }); + it('deleteBucket request should return error if account not ' + 'authorized', done => { + bucketDelete(otherAccountAuthInfo, baseTestRequest, log, err => { + assert.strictEqual(err.is.AccessDenied, true); + done(); + }); }); - it('bucketDeleteWebsite request on bucket with delete flag should return ' + - 'NoSuchBucket error and complete deletion', done => { - bucketDeleteWebsite(authInfo, baseTestRequest, - log, err => { + it( + 'bucketDeleteWebsite request on bucket with delete flag should return ' + + 'NoSuchBucket error and complete deletion', + done => { + bucketDeleteWebsite(authInfo, baseTestRequest, log, err => { assert.strictEqual(err.is.NoSuchBucket, true); confirmDeleted(done); }); - }); + }, + ); - it('bucketGet request on bucket with delete flag should return ' + - 'NoSuchBucket error and complete deletion', done => { - bucketGet(authInfo, baseTestRequest, - log, err => { + it( + 'bucketGet request on bucket with delete flag should return ' + 'NoSuchBucket error and complete deletion', + done => { + bucketGet(authInfo, baseTestRequest, log, err => { assert.strictEqual(err.is.NoSuchBucket, true); confirmDeleted(done); }); - }); + }, + ); - it('bucketGetACL request on bucket with delete flag should return ' + - 'NoSuchBucket error and complete deletion', done => { - bucketGetACL(authInfo, baseTestRequest, - log, err => { + it( + 'bucketGetACL request on bucket with delete flag should return ' + 'NoSuchBucket error and complete deletion', + done => { + bucketGetACL(authInfo, baseTestRequest, log, err => { assert.strictEqual(err.is.NoSuchBucket, true); confirmDeleted(done); }); - }); + }, + ); - it('bucketGetCors request on bucket with delete flag should return ' + - 'NoSuchBucket error and complete deletion', done => { - bucketGetCors(authInfo, baseTestRequest, - log, err => { - assert.strictEqual(err.is.NoSuchBucket, true); - confirmDeleted(done); - }); - }); + it( + 'bucketGetCors request on bucket with delete flag should return ' + 'NoSuchBucket error and complete deletion', + done => { + bucketGetCors(authInfo, baseTestRequest, log, err => { + assert.strictEqual(err.is.NoSuchBucket, true); + confirmDeleted(done); + }); + }, + ); - it('bucketPutCors request on bucket with delete flag should return ' + - 'NoSuchBucket error and complete deletion', done => { - const bucketPutCorsRequest = createAlteredRequest({}, 'headers', - baseTestRequest, baseTestRequest.headers); - bucketPutCorsRequest.post = '' + - 'PUT' + - 'http://www.example.com' + - ''; - bucketPutCorsRequest.headers['content-md5'] = crypto.createHash('md5') - .update(bucketPutCorsRequest.post, 'utf8').digest('base64'); - bucketPutCors(authInfo, bucketPutCorsRequest, log, err => { - assert.strictEqual(err.is.NoSuchBucket, true); - confirmDeleted(done); - }); - }); + it( + 'bucketPutCors request on bucket with delete flag should return ' + 'NoSuchBucket error and complete deletion', + done => { + const bucketPutCorsRequest = createAlteredRequest({}, 'headers', baseTestRequest, baseTestRequest.headers); + bucketPutCorsRequest.post = + '' + + 'PUT' + + 'http://www.example.com' + + ''; + bucketPutCorsRequest.headers['content-md5'] = crypto + .createHash('md5') + .update(bucketPutCorsRequest.post, 'utf8') + .digest('base64'); + bucketPutCors(authInfo, bucketPutCorsRequest, log, err => { + assert.strictEqual(err.is.NoSuchBucket, true); + confirmDeleted(done); + }); + }, + ); - it('bucketDeleteCors request on bucket with delete flag should return ' + - 'NoSuchBucket error and complete deletion', done => { - bucketDeleteCors(authInfo, baseTestRequest, log, err => { - assert.strictEqual(err.is.NoSuchBucket, true); - confirmDeleted(done); - }); - }); + it( + 'bucketDeleteCors request on bucket with delete flag should return ' + + 'NoSuchBucket error and complete deletion', + done => { + bucketDeleteCors(authInfo, baseTestRequest, log, err => { + assert.strictEqual(err.is.NoSuchBucket, true); + confirmDeleted(done); + }); + }, + ); - it('bucketGetWebsite request on bucket with delete flag should return ' + - 'NoSuchBucket error and complete deletion', done => { - bucketGetWebsite(authInfo, baseTestRequest, - log, err => { - assert.strictEqual(err.is.NoSuchBucket, true); - confirmDeleted(done); - }); - }); + it( + 'bucketGetWebsite request on bucket with delete flag should return ' + + 'NoSuchBucket error and complete deletion', + done => { + bucketGetWebsite(authInfo, baseTestRequest, log, err => { + assert.strictEqual(err.is.NoSuchBucket, true); + confirmDeleted(done); + }); + }, + ); - it('bucketPutWebsite request on bucket with delete flag should return ' + - 'NoSuchBucket error and complete deletion', done => { - const bucketPutWebsiteRequest = createAlteredRequest({}, 'headers', - baseTestRequest, baseTestRequest.headers); - bucketPutWebsiteRequest.post = '' + - 'index.html' + - ''; - bucketPutWebsite(authInfo, bucketPutWebsiteRequest, - log, err => { - assert.strictEqual(err.is.NoSuchBucket, true); - confirmDeleted(done); - }); - }); + it( + 'bucketPutWebsite request on bucket with delete flag should return ' + + 'NoSuchBucket error and complete deletion', + done => { + const bucketPutWebsiteRequest = createAlteredRequest( + {}, + 'headers', + baseTestRequest, + baseTestRequest.headers, + ); + bucketPutWebsiteRequest.post = + '' + + 'index.html' + + ''; + bucketPutWebsite(authInfo, bucketPutWebsiteRequest, log, err => { + assert.strictEqual(err.is.NoSuchBucket, true); + confirmDeleted(done); + }); + }, + ); - it('bucketHead request on bucket with delete flag should return ' + - 'NoSuchBucket error and complete deletion', done => { - bucketHead(authInfo, baseTestRequest, - log, err => { + it( + 'bucketHead request on bucket with delete flag should return ' + 'NoSuchBucket error and complete deletion', + done => { + bucketHead(authInfo, baseTestRequest, log, err => { assert.strictEqual(err.is.NoSuchBucket, true); confirmDeleted(done); }); - }); + }, + ); - function checkForNoSuchUploadError(apiAction, partNumber, done, - extraArgNeeded) { - const mpuRequest = createAlteredRequest({}, 'headers', - baseTestRequest, baseTestRequest.headers); + function checkForNoSuchUploadError(apiAction, partNumber, done, extraArgNeeded) { + const mpuRequest = createAlteredRequest({}, 'headers', baseTestRequest, baseTestRequest.headers); const uploadId = '5555'; mpuRequest.objectKey = 'objectName'; mpuRequest.query = { uploadId, partNumber }; if (extraArgNeeded) { - return apiAction(authInfo, mpuRequest, undefined, - log, err => { - assert.strictEqual(err.is.NoSuchUpload, true); - return done(); - }); - } - return apiAction(authInfo, mpuRequest, - log, err => { + return apiAction(authInfo, mpuRequest, undefined, log, err => { assert.strictEqual(err.is.NoSuchUpload, true); return done(); }); + } + return apiAction(authInfo, mpuRequest, log, err => { + assert.strictEqual(err.is.NoSuchUpload, true); + return done(); + }); } - it('completeMultipartUpload request on bucket with deleted flag should ' + - 'return NoSuchUpload error', done => { + it('completeMultipartUpload request on bucket with deleted flag should ' + 'return NoSuchUpload error', done => { checkForNoSuchUploadError(completeMultipartUpload, null, done); }); - it('listParts request on bucket with deleted flag should ' + - 'return NoSuchUpload error', done => { + it('listParts request on bucket with deleted flag should ' + 'return NoSuchUpload error', done => { checkForNoSuchUploadError(listParts, null, done); }); describe('multipartDelete request on a bucket with deleted flag', () => { - it('should return NoSuchUpload error if legacyAWSBehavior is enabled', - done => { - config.locationConstraints[locationConstraint]. - legacyAwsBehavior = true; + it('should return NoSuchUpload error if legacyAWSBehavior is enabled', done => { + config.locationConstraints[locationConstraint].legacyAwsBehavior = true; checkForNoSuchUploadError(multipartDelete, null, done); }); - it('should return no error if legacyAWSBehavior is not enabled', - done => { - config.locationConstraints[locationConstraint]. - legacyAwsBehavior = false; - const mpuRequest = createAlteredRequest({}, 'headers', - baseTestRequest, baseTestRequest.headers); + it('should return no error if legacyAWSBehavior is not enabled', done => { + config.locationConstraints[locationConstraint].legacyAwsBehavior = false; + const mpuRequest = createAlteredRequest({}, 'headers', baseTestRequest, baseTestRequest.headers); const uploadId = '5555'; mpuRequest.objectKey = 'objectName'; mpuRequest.query = { uploadId }; @@ -484,64 +510,61 @@ describe('deleted flag bucket handling', () => { }); }); - it('objectPutPart request on bucket with deleted flag should ' + - 'return NoSuchUpload error', done => { + it('objectPutPart request on bucket with deleted flag should ' + 'return NoSuchUpload error', done => { checkForNoSuchUploadError(objectPutPart, '1', done, true); }); - it('list multipartUploads request on bucket with deleted flag should ' + - 'return NoSuchBucket error', done => { - const listRequest = createAlteredRequest({}, 'headers', - baseTestRequest, baseTestRequest.headers); + it('list multipartUploads request on bucket with deleted flag should ' + 'return NoSuchBucket error', done => { + const listRequest = createAlteredRequest({}, 'headers', baseTestRequest, baseTestRequest.headers); listRequest.query = {}; - listMultipartUploads(authInfo, listRequest, - log, err => { - assert.strictEqual(err.is.NoSuchBucket, true); - done(); - }); + listMultipartUploads(authInfo, listRequest, log, err => { + assert.strictEqual(err.is.NoSuchBucket, true); + done(); + }); }); - it('objectGet request on bucket with deleted flag should' + - 'return NoSuchBucket error and finish deletion', + it( + 'objectGet request on bucket with deleted flag should' + 'return NoSuchBucket error and finish deletion', done => { - objectGet(authInfo, baseTestRequest, false, - log, err => { + objectGet(authInfo, baseTestRequest, false, log, err => { assert.strictEqual(err.is.NoSuchBucket, true); confirmDeleted(done); }); - }); + }, + ); - it('objectGetACL request on bucket with deleted flag should return ' + - 'NoSuchBucket error and complete deletion', done => { - objectGetACL(authInfo, baseTestRequest, - log, err => { - assert.strictEqual(err.is.NoSuchBucket, true); - confirmDeleted(done); - }); - }); + it( + 'objectGetACL request on bucket with deleted flag should return ' + 'NoSuchBucket error and complete deletion', + done => { + objectGetACL(authInfo, baseTestRequest, log, err => { + assert.strictEqual(err.is.NoSuchBucket, true); + confirmDeleted(done); + }); + }, + ); - it('objectHead request on bucket with deleted flag should return ' + - 'NoSuchBucket error and complete deletion', done => { - objectHead(authInfo, baseTestRequest, - log, err => { - assert.strictEqual(err.is.NoSuchBucket, true); - confirmDeleted(done); - }); - }); + it( + 'objectHead request on bucket with deleted flag should return ' + 'NoSuchBucket error and complete deletion', + done => { + objectHead(authInfo, baseTestRequest, log, err => { + assert.strictEqual(err.is.NoSuchBucket, true); + confirmDeleted(done); + }); + }, + ); - it('objectPutACL request on bucket with deleted flag should return ' + - 'NoSuchBucket error and complete deletion', done => { - objectPutACL(authInfo, baseTestRequest, - log, err => { - assert.strictEqual(err.is.NoSuchBucket, true); - confirmDeleted(done); - }); - }); + it( + 'objectPutACL request on bucket with deleted flag should return ' + 'NoSuchBucket error and complete deletion', + done => { + objectPutACL(authInfo, baseTestRequest, log, err => { + assert.strictEqual(err.is.NoSuchBucket, true); + confirmDeleted(done); + }); + }, + ); - it('objectDelete request on bucket with deleted flag should return ' + - 'NoSuchBucket error', done => { - objectDelete(authInfo, baseTestRequest, - log, err => { + it('objectDelete request on bucket with deleted flag should return ' + 'NoSuchBucket error', done => { + objectDelete(authInfo, baseTestRequest, log, err => { assert.strictEqual(err.is.NoSuchBucket, true); confirmDeleted(done); }); diff --git a/tests/unit/api/listMultipartUploads.js b/tests/unit/api/listMultipartUploads.js index e24fc13f40..2ad4777138 100644 --- a/tests/unit/api/listMultipartUploads.js +++ b/tests/unit/api/listMultipartUploads.js @@ -4,8 +4,7 @@ const querystring = require('querystring'); const { parseString } = require('xml2js'); const { bucketPut } = require('../../../lib/api/bucketPut'); -const initiateMultipartUpload - = require('../../../lib/api/initiateMultipartUpload'); +const initiateMultipartUpload = require('../../../lib/api/initiateMultipartUpload'); const listMultipartUploads = require('../../../lib/api/listMultipartUploads'); const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../helpers'); @@ -59,40 +58,39 @@ describe('listMultipartUploads API', () => { actionImplicitDenies: false, }; - it('should return the name of the common prefix ' + - 'of common prefix object keys for multipart uploads if delimiter ' + - 'and prefix specified', done => { - const commonPrefix = `${prefix}${delimiter}`; - const testListRequest = { - bucketName, - namespace, - headers: { host: '/' }, - url: `/${bucketName}?uploads&delimiter=/&prefix=sub`, - query: { delimiter, prefix }, - actionImplicitDenies: false, - }; - - async.waterfall([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - (corsHeaders, next) => initiateMultipartUpload(authInfo, - testInitiateMPURequest1, log, next), - (result, corsHeaders, next) => initiateMultipartUpload(authInfo, - testInitiateMPURequest2, log, next), - (result, corsHeaders, next) => listMultipartUploads(authInfo, - testListRequest, log, next), - (result, corsHeaders, next) => - parseString(result, corsHeaders, next), - ], - (err, result) => { - assert.strictEqual(result.ListMultipartUploadsResult - .CommonPrefixes[0].Prefix[0], - commonPrefix); - done(); - }); - }); - - it('should return list of all multipart uploads if ' + - 'no delimiter specified', done => { + it( + 'should return the name of the common prefix ' + + 'of common prefix object keys for multipart uploads if delimiter ' + + 'and prefix specified', + done => { + const commonPrefix = `${prefix}${delimiter}`; + const testListRequest = { + bucketName, + namespace, + headers: { host: '/' }, + url: `/${bucketName}?uploads&delimiter=/&prefix=sub`, + query: { delimiter, prefix }, + actionImplicitDenies: false, + }; + + async.waterfall( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + (corsHeaders, next) => initiateMultipartUpload(authInfo, testInitiateMPURequest1, log, next), + (result, corsHeaders, next) => + initiateMultipartUpload(authInfo, testInitiateMPURequest2, log, next), + (result, corsHeaders, next) => listMultipartUploads(authInfo, testListRequest, log, next), + (result, corsHeaders, next) => parseString(result, corsHeaders, next), + ], + (err, result) => { + assert.strictEqual(result.ListMultipartUploadsResult.CommonPrefixes[0].Prefix[0], commonPrefix); + done(); + }, + ); + }, + ); + + it('should return list of all multipart uploads if ' + 'no delimiter specified', done => { const testListRequest = { bucketName, namespace, @@ -102,31 +100,24 @@ describe('listMultipartUploads API', () => { actionImplicitDenies: false, }; - - async.waterfall([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - (corsHeaders, next) => initiateMultipartUpload(authInfo, - testInitiateMPURequest1, log, next), - (result, corsHeaders, next) => initiateMultipartUpload(authInfo, - testInitiateMPURequest2, log, next), - (result, corsHeaders, next) => - listMultipartUploads(authInfo, testListRequest, log, next), - (result, corsHeaders, next) => - parseString(result, corsHeaders, next), - ], - (err, result) => { - assert.strictEqual(result.ListMultipartUploadsResult - .Upload[0].Key[0], objectName1); - assert.strictEqual(result.ListMultipartUploadsResult - .Upload[1].Key[0], objectName2); - assert.strictEqual(result.ListMultipartUploadsResult - .IsTruncated[0], 'false'); - done(); - }); + async.waterfall( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + (corsHeaders, next) => initiateMultipartUpload(authInfo, testInitiateMPURequest1, log, next), + (result, corsHeaders, next) => initiateMultipartUpload(authInfo, testInitiateMPURequest2, log, next), + (result, corsHeaders, next) => listMultipartUploads(authInfo, testListRequest, log, next), + (result, corsHeaders, next) => parseString(result, corsHeaders, next), + ], + (err, result) => { + assert.strictEqual(result.ListMultipartUploadsResult.Upload[0].Key[0], objectName1); + assert.strictEqual(result.ListMultipartUploadsResult.Upload[1].Key[0], objectName2); + assert.strictEqual(result.ListMultipartUploadsResult.IsTruncated[0], 'false'); + done(); + }, + ); }); - it('should return no more keys than ' + - 'max-uploads specified', done => { + it('should return no more keys than ' + 'max-uploads specified', done => { const testListRequest = { bucketName, namespace, @@ -136,34 +127,26 @@ describe('listMultipartUploads API', () => { actionImplicitDenies: false, }; - async.waterfall([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - (corsHeaders, next) => initiateMultipartUpload(authInfo, - testInitiateMPURequest1, log, next), - (result, corsHeaders, next) => initiateMultipartUpload(authInfo, - testInitiateMPURequest2, log, next), - (result, corsHeaders, next) => listMultipartUploads(authInfo, - testListRequest, log, next), - (result, corsHeaders, next) => - parseString(result, corsHeaders, next), - ], - (err, result) => { - assert.strictEqual(result.ListMultipartUploadsResult - .Upload[0].Key[0], objectName1); - assert.strictEqual(result.ListMultipartUploadsResult - .Upload[1], undefined); - assert.strictEqual(result.ListMultipartUploadsResult - .IsTruncated[0], 'true'); - assert.strictEqual(result.ListMultipartUploadsResult - .NextKeyMarker[0], objectName1); - assert(result.ListMultipartUploadsResult - .NextUploadIdMarker[0].length > 5); - done(); - }); + async.waterfall( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + (corsHeaders, next) => initiateMultipartUpload(authInfo, testInitiateMPURequest1, log, next), + (result, corsHeaders, next) => initiateMultipartUpload(authInfo, testInitiateMPURequest2, log, next), + (result, corsHeaders, next) => listMultipartUploads(authInfo, testListRequest, log, next), + (result, corsHeaders, next) => parseString(result, corsHeaders, next), + ], + (err, result) => { + assert.strictEqual(result.ListMultipartUploadsResult.Upload[0].Key[0], objectName1); + assert.strictEqual(result.ListMultipartUploadsResult.Upload[1], undefined); + assert.strictEqual(result.ListMultipartUploadsResult.IsTruncated[0], 'true'); + assert.strictEqual(result.ListMultipartUploadsResult.NextKeyMarker[0], objectName1); + assert(result.ListMultipartUploadsResult.NextUploadIdMarker[0].length > 5); + done(); + }, + ); }); - it('should url encode object key name ' + - 'if requested', done => { + it('should url encode object key name ' + 'if requested', done => { const testListRequest = { bucketName, namespace, @@ -173,30 +156,24 @@ describe('listMultipartUploads API', () => { actionImplicitDenies: false, }; - async.waterfall([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - (corsHeaders, next) => initiateMultipartUpload(authInfo, - testInitiateMPURequest1, log, next), - (result, corsHeaders, next) => initiateMultipartUpload(authInfo, - testInitiateMPURequest2, log, next), - (result, corsHeaders, next) => initiateMultipartUpload(authInfo, - testInitiateMPURequest3, log, next), - (result, corsHeaders, next) => listMultipartUploads(authInfo, - testListRequest, log, next), - (result, corsHeaders, next) => - parseString(result, corsHeaders, next), - ], - (err, result) => { - assert.strictEqual(result.ListMultipartUploadsResult - .Upload[0].Key[0], querystring.escape(objectName3)); - assert.strictEqual(result.ListMultipartUploadsResult - .Upload[1].Key[0], querystring.escape(objectName1)); - done(); - }); + async.waterfall( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + (corsHeaders, next) => initiateMultipartUpload(authInfo, testInitiateMPURequest1, log, next), + (result, corsHeaders, next) => initiateMultipartUpload(authInfo, testInitiateMPURequest2, log, next), + (result, corsHeaders, next) => initiateMultipartUpload(authInfo, testInitiateMPURequest3, log, next), + (result, corsHeaders, next) => listMultipartUploads(authInfo, testListRequest, log, next), + (result, corsHeaders, next) => parseString(result, corsHeaders, next), + ], + (err, result) => { + assert.strictEqual(result.ListMultipartUploadsResult.Upload[0].Key[0], querystring.escape(objectName3)); + assert.strictEqual(result.ListMultipartUploadsResult.Upload[1].Key[0], querystring.escape(objectName1)); + done(); + }, + ); }); - it('should return key following specified ' + - 'key-marker', done => { + it('should return key following specified ' + 'key-marker', done => { const testListRequest = { bucketName, namespace, @@ -206,25 +183,20 @@ describe('listMultipartUploads API', () => { actionImplicitDenies: false, }; - async.waterfall([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - (corsHeaders, next) => initiateMultipartUpload(authInfo, - testInitiateMPURequest1, log, next), - (result, corsHeaders, next) => initiateMultipartUpload(authInfo, - testInitiateMPURequest2, log, next), - (result, corsHeaders, next) => initiateMultipartUpload(authInfo, - testInitiateMPURequest3, log, next), - (result, corsHeaders, next) => listMultipartUploads(authInfo, - testListRequest, log, next), - (result, corsHeaders, next) => - parseString(result, corsHeaders, next), - ], - (err, result) => { - assert.strictEqual(result.ListMultipartUploadsResult - .Upload[0].Key[0], objectName2); - assert.strictEqual(result.ListMultipartUploadsResult - .Upload[1], undefined); - done(); - }); + async.waterfall( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + (corsHeaders, next) => initiateMultipartUpload(authInfo, testInitiateMPURequest1, log, next), + (result, corsHeaders, next) => initiateMultipartUpload(authInfo, testInitiateMPURequest2, log, next), + (result, corsHeaders, next) => initiateMultipartUpload(authInfo, testInitiateMPURequest3, log, next), + (result, corsHeaders, next) => listMultipartUploads(authInfo, testListRequest, log, next), + (result, corsHeaders, next) => parseString(result, corsHeaders, next), + ], + (err, result) => { + assert.strictEqual(result.ListMultipartUploadsResult.Upload[0].Key[0], objectName2); + assert.strictEqual(result.ListMultipartUploadsResult.Upload[1], undefined); + done(); + }, + ); }); }); diff --git a/tests/unit/api/listParts.js b/tests/unit/api/listParts.js index 48a9668e11..5cfe7c8394 100644 --- a/tests/unit/api/listParts.js +++ b/tests/unit/api/listParts.js @@ -23,11 +23,9 @@ const mpuBucket = `${constants.mpuBucketPrefix}${bucketName}`; const uploadKey = '$makememulti'; const sixMBObjectETag = '"f3a9fb2071d3503b703938a74eb99846"'; const lastPieceETag = '"555e4cd2f9eff38109d7a3ab13995a32"'; -const overviewKey = `overview${splitter}$makememulti${splitter}4db92ccc-` + - 'd89d-49d3-9fa6-e9c2c1eb31b0'; +const overviewKey = `overview${splitter}$makememulti${splitter}4db92ccc-` + 'd89d-49d3-9fa6-e9c2c1eb31b0'; const partOneKey = `4db92ccc-d89d-49d3-9fa6-e9c2c1eb31b0${splitter}00001`; -const partTwoKey = '4db92ccc-d89d-49d3-9fa6-e9c2c1eb31b0' + - `${splitter}00002`; +const partTwoKey = '4db92ccc-d89d-49d3-9fa6-e9c2c1eb31b0' + `${splitter}00002`; const partThreeKey = `4db92ccc-d89d-49d3-9fa6-e9c2c1eb31b0${splitter}00003`; const partFourKey = `4db92ccc-d89d-49d3-9fa6-e9c2c1eb31b0${splitter}00004`; const partFiveKey = `4db92ccc-d89d-49d3-9fa6-e9c2c1eb31b0${splitter}00005`; @@ -36,74 +34,82 @@ describe('List Parts API', () => { beforeEach(done => { cleanup(); const creationDate = new Date().toJSON(); - const sampleNormalBucketInstance = new BucketInfo(bucketName, - canonicalID, authInfo.getAccountDisplayName(), creationDate, - BucketInfo.currentModelVersion()); - const sampleMPUInstance = new BucketInfo(mpuBucket, - 'admin', 'admin', creationDate, BucketInfo.currentModelVersion()); - metadata.createBucket(bucketName, sampleNormalBucketInstance, log, - () => { - metadata.createBucket(mpuBucket, sampleMPUInstance, log, () => { - inMemMetadata.keyMaps.get(mpuBucket).set(overviewKey, { - 'id': '4db92ccc-d89d-49d3-9fa6-e9c2c1eb31b0', - 'owner-display-name': authInfo.getAccountDisplayName(), - 'owner-id': canonicalID, - 'initiator': { - DisplayName: authInfo.getAccountDisplayName(), - ID: canonicalID, - }, - 'key': '$makememulti', - 'initiated': '2015-11-30T22:40:07.858Z', - 'uploadId': '4db92ccc-d89d-49d3-9fa6-e9c2c1eb31b0', - 'acl': { - Canned: 'private', - FULL_CONTROL: [], - WRITE_ACP: [], - READ: [], - READ_ACP: [], - }, - 'eventualStorageBucket': 'freshestbucket', - 'mdBucketModelVersion': 2, - }); + const sampleNormalBucketInstance = new BucketInfo( + bucketName, + canonicalID, + authInfo.getAccountDisplayName(), + creationDate, + BucketInfo.currentModelVersion(), + ); + const sampleMPUInstance = new BucketInfo( + mpuBucket, + 'admin', + 'admin', + creationDate, + BucketInfo.currentModelVersion(), + ); + metadata.createBucket(bucketName, sampleNormalBucketInstance, log, () => { + metadata.createBucket(mpuBucket, sampleMPUInstance, log, () => { + inMemMetadata.keyMaps.get(mpuBucket).set(overviewKey, { + id: '4db92ccc-d89d-49d3-9fa6-e9c2c1eb31b0', + 'owner-display-name': authInfo.getAccountDisplayName(), + 'owner-id': canonicalID, + initiator: { + DisplayName: authInfo.getAccountDisplayName(), + ID: canonicalID, + }, + key: '$makememulti', + initiated: '2015-11-30T22:40:07.858Z', + uploadId: '4db92ccc-d89d-49d3-9fa6-e9c2c1eb31b0', + acl: { + Canned: 'private', + FULL_CONTROL: [], + WRITE_ACP: [], + READ: [], + READ_ACP: [], + }, + eventualStorageBucket: 'freshestbucket', + mdBucketModelVersion: 2, + }); - inMemMetadata.keyMaps.get(mpuBucket).set(partOneKey, { - 'key': partOneKey, - 'last-modified': '2015-11-30T22:41:18.658Z', - 'content-md5': 'f3a9fb2071d3503b703938a74eb99846', - 'content-length': '6000000', - 'partLocations': ['068db6a6745a79d54c1b29ff99f9f131'], - }); - inMemMetadata.keyMaps.get(mpuBucket).set(partTwoKey, { - 'key': partTwoKey, - 'last-modified': '2015-11-30T22:41:40.207Z', - 'content-md5': 'f3a9fb2071d3503b703938a74eb99846', - 'content-length': '6000000', - 'partLocations': ['ff22f316b16956ff5118c93abce7d62d'], - }); - inMemMetadata.keyMaps.get(mpuBucket).set(partThreeKey, { - 'key': partThreeKey, - 'last-modified': '2015-11-30T22:41:52.102Z', - 'content-md5': 'f3a9fb2071d3503b703938a74eb99846', - 'content-length': '6000000', - 'partLocations': ['dea282f70edb6fc5f9433cd6f525d4a6'], - }); - inMemMetadata.keyMaps.get(mpuBucket).set(partFourKey, { - 'key': partFourKey, - 'last-modified': '2015-11-30T22:42:03.493Z', - 'content-md5': 'f3a9fb2071d3503b703938a74eb99846', - 'content-length': '6000000', - 'partLocations': ['afe24bc40153982e1f7f28066f7af6a4'], - }); - inMemMetadata.keyMaps.get(mpuBucket).set(partFiveKey, { - 'key': partFiveKey, - 'last-modified': '2015-11-30T22:42:22.876Z', - 'content-md5': '555e4cd2f9eff38109d7a3ab13995a32', - 'content-length': '18', - 'partLocations': ['85bc16f5769687070fb13cfe66b5e41f'], - }); - done(); + inMemMetadata.keyMaps.get(mpuBucket).set(partOneKey, { + key: partOneKey, + 'last-modified': '2015-11-30T22:41:18.658Z', + 'content-md5': 'f3a9fb2071d3503b703938a74eb99846', + 'content-length': '6000000', + partLocations: ['068db6a6745a79d54c1b29ff99f9f131'], + }); + inMemMetadata.keyMaps.get(mpuBucket).set(partTwoKey, { + key: partTwoKey, + 'last-modified': '2015-11-30T22:41:40.207Z', + 'content-md5': 'f3a9fb2071d3503b703938a74eb99846', + 'content-length': '6000000', + partLocations: ['ff22f316b16956ff5118c93abce7d62d'], + }); + inMemMetadata.keyMaps.get(mpuBucket).set(partThreeKey, { + key: partThreeKey, + 'last-modified': '2015-11-30T22:41:52.102Z', + 'content-md5': 'f3a9fb2071d3503b703938a74eb99846', + 'content-length': '6000000', + partLocations: ['dea282f70edb6fc5f9433cd6f525d4a6'], }); + inMemMetadata.keyMaps.get(mpuBucket).set(partFourKey, { + key: partFourKey, + 'last-modified': '2015-11-30T22:42:03.493Z', + 'content-md5': 'f3a9fb2071d3503b703938a74eb99846', + 'content-length': '6000000', + partLocations: ['afe24bc40153982e1f7f28066f7af6a4'], + }); + inMemMetadata.keyMaps.get(mpuBucket).set(partFiveKey, { + key: partFiveKey, + 'last-modified': '2015-11-30T22:42:22.876Z', + 'content-md5': '555e4cd2f9eff38109d7a3ab13995a32', + 'content-length': '18', + partLocations: ['85bc16f5769687070fb13cfe66b5e41f'], + }); + done(); }); + }); }); it('should list all parts of a multipart upload', done => { @@ -125,24 +131,15 @@ describe('List Parts API', () => { assert.strictEqual(json.ListPartsResult.Key[0], uploadKey); assert.strictEqual(json.ListPartsResult.UploadId[0], uploadId); assert.strictEqual(json.ListPartsResult.MaxParts[0], '1000'); - assert.strictEqual(json.ListPartsResult.Initiator[0].ID[0], - authInfo.getCanonicalID()); - assert.strictEqual(json.ListPartsResult.IsTruncated[0], - 'false'); - assert.strictEqual(json.ListPartsResult.PartNumberMarker, - undefined); - assert.strictEqual(json.ListPartsResult.NextPartNumberMarker, - undefined); - assert.strictEqual(json.ListPartsResult.Part[0].PartNumber[0], - '1'); - assert.strictEqual(json.ListPartsResult.Part[0].ETag[0], - sixMBObjectETag); - assert.strictEqual(json.ListPartsResult.Part[0].Size[0], - '6000000'); - assert.strictEqual(json.ListPartsResult.Part[4].PartNumber[0], - '5'); - assert.strictEqual(json.ListPartsResult.Part[4].ETag[0], - lastPieceETag); + assert.strictEqual(json.ListPartsResult.Initiator[0].ID[0], authInfo.getCanonicalID()); + assert.strictEqual(json.ListPartsResult.IsTruncated[0], 'false'); + assert.strictEqual(json.ListPartsResult.PartNumberMarker, undefined); + assert.strictEqual(json.ListPartsResult.NextPartNumberMarker, undefined); + assert.strictEqual(json.ListPartsResult.Part[0].PartNumber[0], '1'); + assert.strictEqual(json.ListPartsResult.Part[0].ETag[0], sixMBObjectETag); + assert.strictEqual(json.ListPartsResult.Part[0].Size[0], '6000000'); + assert.strictEqual(json.ListPartsResult.Part[4].PartNumber[0], '5'); + assert.strictEqual(json.ListPartsResult.Part[4].ETag[0], lastPieceETag); assert.strictEqual(json.ListPartsResult.Part[4].Size[0], '18'); assert.strictEqual(json.ListPartsResult.Part.length, 5); done(); @@ -168,15 +165,13 @@ describe('List Parts API', () => { listParts(authInfo, listRequest, log, (err, xml) => { assert.strictEqual(err, null); parseString(xml, (err, json) => { - assert.strictEqual(json.ListPartsResult.Key[0], - urlEncodedObjectKey); + assert.strictEqual(json.ListPartsResult.Key[0], urlEncodedObjectKey); done(); }); }); }); - it('should list only up to requested number ' + - 'of max parts of a multipart upload', done => { + it('should list only up to requested number ' + 'of max parts of a multipart upload', done => { const listRequest = { bucketName, namespace, @@ -198,27 +193,20 @@ describe('List Parts API', () => { assert.strictEqual(json.ListPartsResult.Key[0], uploadKey); assert.strictEqual(json.ListPartsResult.UploadId[0], uploadId); assert.strictEqual(json.ListPartsResult.MaxParts[0], '4'); - assert.strictEqual(json.ListPartsResult.Initiator[0].ID[0], - authInfo.getCanonicalID()); + assert.strictEqual(json.ListPartsResult.Initiator[0].ID[0], authInfo.getCanonicalID()); assert.strictEqual(json.ListPartsResult.IsTruncated[0], 'true'); - assert.strictEqual(json.ListPartsResult.PartNumberMarker, - undefined); - assert.strictEqual(json.ListPartsResult.NextPartNumberMarker[0], - '4'); - assert.strictEqual(json.ListPartsResult.Part[2].PartNumber[0], - '3'); - assert.strictEqual(json.ListPartsResult.Part[2].ETag[0], - sixMBObjectETag); - assert.strictEqual(json.ListPartsResult.Part[2].Size[0], - '6000000'); + assert.strictEqual(json.ListPartsResult.PartNumberMarker, undefined); + assert.strictEqual(json.ListPartsResult.NextPartNumberMarker[0], '4'); + assert.strictEqual(json.ListPartsResult.Part[2].PartNumber[0], '3'); + assert.strictEqual(json.ListPartsResult.Part[2].ETag[0], sixMBObjectETag); + assert.strictEqual(json.ListPartsResult.Part[2].Size[0], '6000000'); assert.strictEqual(json.ListPartsResult.Part.length, 4); done(); }); }); }); - it('should list all parts if requested max-parts ' + - 'is greater than total number of parts', done => { + it('should list all parts if requested max-parts ' + 'is greater than total number of parts', done => { const listRequest = { bucketName, namespace, @@ -240,20 +228,13 @@ describe('List Parts API', () => { assert.strictEqual(json.ListPartsResult.Key[0], uploadKey); assert.strictEqual(json.ListPartsResult.UploadId[0], uploadId); assert.strictEqual(json.ListPartsResult.MaxParts[0], '6'); - assert.strictEqual(json.ListPartsResult.Initiator[0].ID[0], - authInfo.getCanonicalID()); - assert.strictEqual(json.ListPartsResult.IsTruncated[0], - 'false'); - assert.strictEqual(json.ListPartsResult.PartNumberMarker, - undefined); - assert.strictEqual(json.ListPartsResult.NextPartNumberMarker, - undefined); - assert.strictEqual(json.ListPartsResult.Part[2].PartNumber[0], - '3'); - assert.strictEqual(json.ListPartsResult.Part[2].ETag[0], - sixMBObjectETag); - assert.strictEqual(json.ListPartsResult.Part[2].Size[0], - '6000000'); + assert.strictEqual(json.ListPartsResult.Initiator[0].ID[0], authInfo.getCanonicalID()); + assert.strictEqual(json.ListPartsResult.IsTruncated[0], 'false'); + assert.strictEqual(json.ListPartsResult.PartNumberMarker, undefined); + assert.strictEqual(json.ListPartsResult.NextPartNumberMarker, undefined); + assert.strictEqual(json.ListPartsResult.Part[2].PartNumber[0], '3'); + assert.strictEqual(json.ListPartsResult.Part[2].ETag[0], sixMBObjectETag); + assert.strictEqual(json.ListPartsResult.Part[2].Size[0], '6000000'); assert.strictEqual(json.ListPartsResult.Part.length, 5); done(); }); @@ -282,30 +263,21 @@ describe('List Parts API', () => { assert.strictEqual(json.ListPartsResult.Key[0], uploadKey); assert.strictEqual(json.ListPartsResult.UploadId[0], uploadId); assert.strictEqual(json.ListPartsResult.MaxParts[0], '1000'); - assert.strictEqual(json.ListPartsResult.Initiator[0].ID[0], - authInfo.getCanonicalID()); - assert.strictEqual(json.ListPartsResult.IsTruncated[0], - 'false'); - assert.strictEqual(json.ListPartsResult.PartNumberMarker[0], - '2'); - assert.strictEqual(json.ListPartsResult.NextPartNumberMarker, - undefined); - assert.strictEqual(json.ListPartsResult.Part[0].PartNumber[0], - '3'); - assert.strictEqual(json.ListPartsResult.Part[0].ETag[0], - sixMBObjectETag); - assert.strictEqual(json.ListPartsResult.Part[0].Size[0], - '6000000'); - assert.strictEqual(json.ListPartsResult.Part[2].PartNumber[0], - '5'); + assert.strictEqual(json.ListPartsResult.Initiator[0].ID[0], authInfo.getCanonicalID()); + assert.strictEqual(json.ListPartsResult.IsTruncated[0], 'false'); + assert.strictEqual(json.ListPartsResult.PartNumberMarker[0], '2'); + assert.strictEqual(json.ListPartsResult.NextPartNumberMarker, undefined); + assert.strictEqual(json.ListPartsResult.Part[0].PartNumber[0], '3'); + assert.strictEqual(json.ListPartsResult.Part[0].ETag[0], sixMBObjectETag); + assert.strictEqual(json.ListPartsResult.Part[0].Size[0], '6000000'); + assert.strictEqual(json.ListPartsResult.Part[2].PartNumber[0], '5'); assert.strictEqual(json.ListPartsResult.Part.length, 3); done(); }); }); }); - it('should handle a part-number-marker specified ' + - 'and a max-parts specified', done => { + it('should handle a part-number-marker specified ' + 'and a max-parts specified', done => { const listRequest = { bucketName, namespace, @@ -328,21 +300,14 @@ describe('List Parts API', () => { assert.strictEqual(json.ListPartsResult.Key[0], uploadKey); assert.strictEqual(json.ListPartsResult.UploadId[0], uploadId); assert.strictEqual(json.ListPartsResult.MaxParts[0], '2'); - assert.strictEqual(json.ListPartsResult.Initiator[0].ID[0], - authInfo.getCanonicalID()); + assert.strictEqual(json.ListPartsResult.Initiator[0].ID[0], authInfo.getCanonicalID()); assert.strictEqual(json.ListPartsResult.IsTruncated[0], 'true'); - assert.strictEqual(json.ListPartsResult.PartNumberMarker[0], - '2'); - assert.strictEqual(json.ListPartsResult.NextPartNumberMarker[0], - '4'); - assert.strictEqual(json.ListPartsResult.Part[0].PartNumber[0], - '3'); - assert.strictEqual(json.ListPartsResult.Part[0].ETag[0], - sixMBObjectETag); - assert.strictEqual(json.ListPartsResult.Part[0].Size[0], - '6000000'); - assert.strictEqual(json.ListPartsResult.Part[1].PartNumber[0], - '4'); + assert.strictEqual(json.ListPartsResult.PartNumberMarker[0], '2'); + assert.strictEqual(json.ListPartsResult.NextPartNumberMarker[0], '4'); + assert.strictEqual(json.ListPartsResult.Part[0].PartNumber[0], '3'); + assert.strictEqual(json.ListPartsResult.Part[0].ETag[0], sixMBObjectETag); + assert.strictEqual(json.ListPartsResult.Part[0].Size[0], '6000000'); + assert.strictEqual(json.ListPartsResult.Part[1].PartNumber[0], '4'); assert.strictEqual(json.ListPartsResult.Part.length, 2); done(); }); diff --git a/tests/unit/api/multipartDelete.js b/tests/unit/api/multipartDelete.js index be9e9ec81e..dae41d6491 100644 --- a/tests/unit/api/multipartDelete.js +++ b/tests/unit/api/multipartDelete.js @@ -6,8 +6,7 @@ const { cleanup, DummyRequestLogger } = require('../helpers'); const { config } = require('../../../lib/Config'); const DummyRequest = require('../DummyRequest'); const { bucketPut } = require('../../../lib/api/bucketPut'); -const initiateMultipartUpload - = require('../../../lib/api/initiateMultipartUpload'); +const initiateMultipartUpload = require('../../../lib/api/initiateMultipartUpload'); const multipartDelete = require('../../../lib/api/multipartDelete'); const objectPutPart = require('../../../lib/api/objectPutPart'); const { makeAuthInfo } = require('../helpers'); @@ -36,62 +35,63 @@ const initiateRequest = { const eastLocation = 'us-east-1'; const westLocation = 'scality-internal-file'; -function _createAndAbortMpu(usEastSetting, fakeUploadID, locationConstraint, - callback) { - config.locationConstraints['us-east-1'].legacyAwsBehavior = - usEastSetting; - const post = '' + +function _createAndAbortMpu(usEastSetting, fakeUploadID, locationConstraint, callback) { + config.locationConstraints['us-east-1'].legacyAwsBehavior = usEastSetting; + const post = + '' + '' + `${locationConstraint}` + ''; const testBucketPutRequest = Object.assign({ post }, bucketPutRequest); - async.waterfall([ - next => bucketPut(authInfo, testBucketPutRequest, log, next), - (corsHeaders, next) => - initiateMultipartUpload(authInfo, initiateRequest, log, next), - (result, corsHeaders, next) => parseString(result, next), - (json, next) => { - // use uploadId parsed from initiateMpu request to construct - // uploadPart and deleteMpu requests - const uploadId = - json.InitiateMultipartUploadResult.UploadId[0]; - const partBody = Buffer.from('I am a part\n', 'utf8'); - const partRequest = new DummyRequest({ - bucketName, - namespace, - objectKey, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - url: `/${objectKey}?partNumber=1&uploadId=${uploadId}`, - query: { - partNumber: '1', - uploadId, - }, - actionImplicitDenies: false, - }, partBody); - const testUploadId = fakeUploadID ? 'nonexistinguploadid' : - uploadId; - const deleteMpuRequest = { - bucketName, - namespace, - objectKey, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - url: `/${objectKey}?uploadId=${testUploadId}`, - query: { uploadId: testUploadId }, - actionImplicitDenies: false, - }; - next(null, partRequest, deleteMpuRequest); - }, - (partRequest, deleteMpuRequest, next) => - objectPutPart(authInfo, partRequest, undefined, log, err => { - if (err) { - return next(err); - } - return next(null, deleteMpuRequest); - }), - (deleteMpuRequest, next) => - multipartDelete(authInfo, deleteMpuRequest, log, next), - ], callback); + async.waterfall( + [ + next => bucketPut(authInfo, testBucketPutRequest, log, next), + (corsHeaders, next) => initiateMultipartUpload(authInfo, initiateRequest, log, next), + (result, corsHeaders, next) => parseString(result, next), + (json, next) => { + // use uploadId parsed from initiateMpu request to construct + // uploadPart and deleteMpu requests + const uploadId = json.InitiateMultipartUploadResult.UploadId[0]; + const partBody = Buffer.from('I am a part\n', 'utf8'); + const partRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: `/${objectKey}?partNumber=1&uploadId=${uploadId}`, + query: { + partNumber: '1', + uploadId, + }, + actionImplicitDenies: false, + }, + partBody, + ); + const testUploadId = fakeUploadID ? 'nonexistinguploadid' : uploadId; + const deleteMpuRequest = { + bucketName, + namespace, + objectKey, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: `/${objectKey}?uploadId=${testUploadId}`, + query: { uploadId: testUploadId }, + actionImplicitDenies: false, + }; + next(null, partRequest, deleteMpuRequest); + }, + (partRequest, deleteMpuRequest, next) => + objectPutPart(authInfo, partRequest, undefined, log, err => { + if (err) { + return next(err); + } + return next(null, deleteMpuRequest); + }), + (deleteMpuRequest, next) => multipartDelete(authInfo, deleteMpuRequest, log, next), + ], + callback, + ); } describe('Multipart Delete API', () => { @@ -100,41 +100,47 @@ describe('Multipart Delete API', () => { }); afterEach(() => { // set back to original - config.locationConstraints['us-east-1'].legacyAwsBehavior = - true; + config.locationConstraints['us-east-1'].legacyAwsBehavior = true; cleanup(); }); - it('should not return error if mpu exists with uploadId and at least ' + - 'one part', done => { + it('should not return error if mpu exists with uploadId and at least ' + 'one part', done => { _createAndAbortMpu(true, false, eastLocation, err => { assert.ifError(err); done(err); }); }); - it('should still not return error if uploadId does not exist on ' + - 'multipart abort call, in region other than us-east-1', done => { - _createAndAbortMpu(true, true, westLocation, err => { - assert.ifError(err); - done(err); - }); - }); + it( + 'should still not return error if uploadId does not exist on ' + + 'multipart abort call, in region other than us-east-1', + done => { + _createAndAbortMpu(true, true, westLocation, err => { + assert.ifError(err); + done(err); + }); + }, + ); - it('bucket created in us-east-1: should return 404 if uploadId does not ' + - 'exist and legacyAwsBehavior set to true', - done => { - _createAndAbortMpu(true, true, eastLocation, err => { - assert.strictEqual(err.is.NoSuchUpload, true); - done(); - }); - }); + it( + 'bucket created in us-east-1: should return 404 if uploadId does not ' + + 'exist and legacyAwsBehavior set to true', + done => { + _createAndAbortMpu(true, true, eastLocation, err => { + assert.strictEqual(err.is.NoSuchUpload, true); + done(); + }); + }, + ); - it('bucket created in us-east-1: should return no error ' + - 'if uploadId does not exist and legacyAwsBehavior set to false', done => { - _createAndAbortMpu(false, true, eastLocation, err => { - assert.strictEqual(err, null, `Expected no error, got ${err}`); - done(); - }); - }); + it( + 'bucket created in us-east-1: should return no error ' + + 'if uploadId does not exist and legacyAwsBehavior set to false', + done => { + _createAndAbortMpu(false, true, eastLocation, err => { + assert.strictEqual(err, null, `Expected no error, got ${err}`); + done(); + }); + }, + ); }); diff --git a/tests/unit/api/multipartUpload.js b/tests/unit/api/multipartUpload.js index 75bdc1b2e2..73f0c55901 100644 --- a/tests/unit/api/multipartUpload.js +++ b/tests/unit/api/multipartUpload.js @@ -12,15 +12,12 @@ const { bucketPut } = require('../../../lib/api/bucketPut'); const bucketPutPolicy = require('../../../lib/api/bucketPutPolicy'); const bucketPutVersioning = require('../../../lib/api/bucketPutVersioning'); const objectPut = require('../../../lib/api/objectPut'); -const completeMultipartUpload - = require('../../../lib/api/completeMultipartUpload'); +const completeMultipartUpload = require('../../../lib/api/completeMultipartUpload'); const constants = require('../../../constants'); -const { cleanup, DummyRequestLogger, makeAuthInfo, versioningTestUtils } - = require('../helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo, versioningTestUtils } = require('../helpers'); const getObjectLegalHold = require('../../../lib/api/objectGetLegalHold'); const getObjectRetention = require('../../../lib/api/objectGetRetention'); -const initiateMultipartUpload - = require('../../../lib/api/initiateMultipartUpload'); +const initiateMultipartUpload = require('../../../lib/api/initiateMultipartUpload'); const multipartDelete = require('../../../lib/api/multipartDelete'); const objectPutPart = require('../../../lib/api/objectPutPart'); const DummyRequest = require('../DummyRequest'); @@ -29,9 +26,7 @@ const metadataswitch = require('../metadataswitch'); const { fakeMetadataArchive } = require('../../functional/aws-node-sdk/test/utils/init'); const { config } = require('../../../lib/Config'); -const { - LOCATION_NAME_CRR, -} = require('../../constants'); +const { LOCATION_NAME_CRR } = require('../../constants'); const { data } = require('../../../lib/data/wrapper'); const { metadata } = storage.metadata.inMemory.metadata; @@ -56,16 +51,17 @@ const bucketPutRequest = { namespace, headers: { host: `${bucketName}.s3.amazonaws.com` }, url: '/', - post: '' + - 'scality-internal-mem' + - '', + post: + '' + + 'scality-internal-mem' + + '', actionImplicitDenies: false, }; const lockEnabledBucketRequest = Object.assign({}, bucketPutRequest); lockEnabledBucketRequest.bucketName = lockedBucket; lockEnabledBucketRequest.headers = { - 'host': `${lockedBucket}.s3.amazonaws.com`, + host: `${lockedBucket}.s3.amazonaws.com`, 'x-amz-bucket-object-lock-enabled': 'true', }; const initiateRequest = { @@ -84,13 +80,13 @@ retentionInitiateRequest.bucketName = lockedBucket; retentionInitiateRequest.headers = { 'x-amz-object-lock-mode': 'GOVERNANCE', 'x-amz-object-lock-retain-until-date': futureDate, - 'host': `${lockedBucket}.s3.amazonaws.com`, + host: `${lockedBucket}.s3.amazonaws.com`, }; const legalHoldInitiateRequest = Object.assign({}, initiateRequest); legalHoldInitiateRequest.bucketName = lockedBucket; legalHoldInitiateRequest.headers = { 'x-amz-object-lock-legal-hold': 'ON', - 'host': `${lockedBucket}.s3.amazonaws.com`, + host: `${lockedBucket}.s3.amazonaws.com`, }; const getObjectLockInfoRequest = { @@ -112,29 +108,31 @@ const expectedLegalHold = { function _createPutPartRequest(uploadId, partNumber, partBody) { const md5Hash = crypto.createHash('md5').update(partBody); const partHash = md5Hash.digest('hex'); - return new DummyRequest({ - bucketName, - namespace, - objectKey, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - url: `/${objectKey}?partNumber=${partNumber}&uploadId=${uploadId}`, - query: { - partNumber, - uploadId, + return new DummyRequest( + { + bucketName, + namespace, + objectKey, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: `/${objectKey}?partNumber=${partNumber}&uploadId=${uploadId}`, + query: { + partNumber, + uploadId, + }, + partHash, + actionImplicitDenies: false, }, - partHash, - actionImplicitDenies: false, - }, partBody); + partBody, + ); } function _createCompleteMpuRequest(uploadId, parts) { const completeBody = []; completeBody.push(''); parts.forEach(part => { - completeBody.push('' + - `${part.partNumber}` + - `"${part.eTag}"` + - ''); + completeBody.push( + '' + `${part.partNumber}` + `"${part.eTag}"` + '', + ); }); completeBody.push(''); return { @@ -159,8 +157,8 @@ async function _uploadMpuObject(params = {}) { return json.InitiateMultipartUploadResult.UploadId[0]; }; const _objectPutPart = util.promisify(objectPutPart); - const _completeMultipartUpload = (...params) => util.promisify(cb => - completeMultipartUpload(...params, (err, xml, headers) => cb(err, { xml, headers })))(); + const _completeMultipartUpload = (...params) => + util.promisify(cb => completeMultipartUpload(...params, (err, xml, headers) => cb(err, { xml, headers })))(); const headers = { ...initiateRequest.headers }; if (params.location) { @@ -194,41 +192,33 @@ describe('Multipart Upload API', () => { }); it('mpuBucketPrefix should be a defined constant', () => { - assert(constants.mpuBucketPrefix, - 'Expected mpuBucketPrefix to be defined'); + assert(constants.mpuBucketPrefix, 'Expected mpuBucketPrefix to be defined'); }); it('should initiate a multipart upload', done => { bucketPut(authInfo, bucketPutRequest, log, err => { assert.ifError(err); - initiateMultipartUpload(authInfo, initiateRequest, - log, (err, result) => { - assert.ifError(err); - parseString(result, (err, json) => { - assert.strictEqual(json.InitiateMultipartUploadResult - .Bucket[0], bucketName); - assert.strictEqual(json.InitiateMultipartUploadResult - .Key[0], objectKey); - assert(json.InitiateMultipartUploadResult.UploadId[0]); - assert(metadata.buckets.get(mpuBucket)._name, - mpuBucket); - const mpuKeys = metadata.keyMaps.get(mpuBucket); - assert.strictEqual(mpuKeys.size, 1); - assert(mpuKeys.keys().next().value - .startsWith(`overview${splitter}${objectKey}`)); - done(); - }); + initiateMultipartUpload(authInfo, initiateRequest, log, (err, result) => { + assert.ifError(err); + parseString(result, (err, json) => { + assert.strictEqual(json.InitiateMultipartUploadResult.Bucket[0], bucketName); + assert.strictEqual(json.InitiateMultipartUploadResult.Key[0], objectKey); + assert(json.InitiateMultipartUploadResult.UploadId[0]); + assert(metadata.buckets.get(mpuBucket)._name, mpuBucket); + const mpuKeys = metadata.keyMaps.get(mpuBucket); + assert.strictEqual(mpuKeys.size, 1); + assert(mpuKeys.keys().next().value.startsWith(`overview${splitter}${objectKey}`)); + done(); }); + }); }); }); - it('should return an error on an initiate multipart upload call if ' + - 'no destination bucket', done => { - initiateMultipartUpload(authInfo, initiateRequest, - log, err => { - assert(err.is.NoSuchBucket); - done(); - }); + it('should return an error on an initiate multipart upload call if ' + 'no destination bucket', done => { + initiateMultipartUpload(authInfo, initiateRequest, log, err => { + assert(err.is.NoSuchBucket); + done(); + }); }); it('should not mpu with storage-class header not equal to STANDARD', done => { @@ -242,664 +232,542 @@ describe('Multipart Upload API', () => { }, url: `/${objectKey}?uploads`, }; - initiateMultipartUpload(authInfo, initiateRequestCold, - log, err => { - assert.strictEqual(err.is.InvalidStorageClass, true); - done(); - }); + initiateMultipartUpload(authInfo, initiateRequestCold, log, err => { + assert.strictEqual(err.is.InvalidStorageClass, true); + done(); + }); }); it('should upload a part', done => { - async.waterfall([ - next => bucketPut(authInfo, bucketPutRequest, log, next), - (corsHeaders, next) => initiateMultipartUpload(authInfo, - initiateRequest, log, next), - (result, corsHeaders, next) => { - const mpuKeys = metadata.keyMaps.get(mpuBucket); - assert.strictEqual(mpuKeys.size, 1); - assert(mpuKeys.keys().next().value - .startsWith(`overview${splitter}${objectKey}`)); - parseString(result, next); - }, - ], - (err, json) => { - // Need to build request in here since do not have uploadId - // until here - assert.ifError(err); - const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; - const md5Hash = crypto.createHash('md5'); - const bufferBody = Buffer.from(postBody); - md5Hash.update(bufferBody); - const partHash = md5Hash.digest('hex'); - const partRequest = new DummyRequest({ - bucketName, - objectKey, - namespace, - url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - query: { - partNumber: '1', - uploadId: testUploadId, + async.waterfall( + [ + next => bucketPut(authInfo, bucketPutRequest, log, next), + (corsHeaders, next) => initiateMultipartUpload(authInfo, initiateRequest, log, next), + (result, corsHeaders, next) => { + const mpuKeys = metadata.keyMaps.get(mpuBucket); + assert.strictEqual(mpuKeys.size, 1); + assert(mpuKeys.keys().next().value.startsWith(`overview${splitter}${objectKey}`)); + parseString(result, next); }, - partHash, - }, postBody); - objectPutPart(authInfo, partRequest, undefined, log, err => { + ], + (err, json) => { + // Need to build request in here since do not have uploadId + // until here assert.ifError(err); - const keysInMPUkeyMap = []; - metadata.keyMaps.get(mpuBucket).forEach((val, key) => { - keysInMPUkeyMap.push(key); - }); - const sortedKeyMap = keysInMPUkeyMap.sort(a => { - if (a.slice(0, 8) === 'overview') { - return -1; - } - return 0; + const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; + const md5Hash = crypto.createHash('md5'); + const bufferBody = Buffer.from(postBody); + md5Hash.update(bufferBody); + const partHash = md5Hash.digest('hex'); + const partRequest = new DummyRequest( + { + bucketName, + objectKey, + namespace, + url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + query: { + partNumber: '1', + uploadId: testUploadId, + }, + partHash, + }, + postBody, + ); + objectPutPart(authInfo, partRequest, undefined, log, err => { + assert.ifError(err); + const keysInMPUkeyMap = []; + metadata.keyMaps.get(mpuBucket).forEach((val, key) => { + keysInMPUkeyMap.push(key); + }); + const sortedKeyMap = keysInMPUkeyMap.sort(a => { + if (a.slice(0, 8) === 'overview') { + return -1; + } + return 0; + }); + const overviewEntry = sortedKeyMap[0]; + const partKey = sortedKeyMap[1]; + const partEntryArray = partKey.split(splitter); + const partUploadId = partEntryArray[0]; + const firstPartNumber = partEntryArray[1]; + const partETag = metadata.keyMaps.get(mpuBucket).get(partKey)['content-md5']; + assert.strictEqual(keysInMPUkeyMap.length, 2); + assert.strictEqual(metadata.keyMaps.get(mpuBucket).get(overviewEntry).key, objectKey); + assert.strictEqual(partUploadId, testUploadId); + assert.strictEqual(firstPartNumber, '00001'); + assert.strictEqual(partETag, partHash); + done(); }); - const overviewEntry = sortedKeyMap[0]; - const partKey = sortedKeyMap[1]; - const partEntryArray = partKey.split(splitter); - const partUploadId = partEntryArray[0]; - const firstPartNumber = partEntryArray[1]; - const partETag = metadata.keyMaps.get(mpuBucket) - .get(partKey)['content-md5']; - assert.strictEqual(keysInMPUkeyMap.length, 2); - assert.strictEqual(metadata.keyMaps.get(mpuBucket) - .get(overviewEntry).key, - objectKey); - assert.strictEqual(partUploadId, testUploadId); - assert.strictEqual(firstPartNumber, '00001'); - assert.strictEqual(partETag, partHash); - done(); - }); - }); + }, + ); }); - it('should upload a part even if the client sent a base 64 ETag ' + - '(and the stored ETag in metadata should be hex)', done => { - async.waterfall([ - next => bucketPut(authInfo, bucketPutRequest, log, next), - (corsHeaders, next) => initiateMultipartUpload(authInfo, - initiateRequest, log, next), - (result, corsHeaders, next) => parseString(result, next), - ], - (err, json) => { - // Need to build request in here since do not have uploadId - // until here - assert.ifError(err); - const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; - const md5Hash = crypto.createHash('md5'); - const bufferBody = Buffer.from(postBody); - const partHash = md5Hash.update(bufferBody).digest('hex'); - const partRequest = new DummyRequest({ - bucketName, - namespace, - objectKey, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, - query: { - partNumber: '1', - uploadId: testUploadId, + it( + 'should upload a part even if the client sent a base 64 ETag ' + + '(and the stored ETag in metadata should be hex)', + done => { + async.waterfall( + [ + next => bucketPut(authInfo, bucketPutRequest, log, next), + (corsHeaders, next) => initiateMultipartUpload(authInfo, initiateRequest, log, next), + (result, corsHeaders, next) => parseString(result, next), + ], + (err, json) => { + // Need to build request in here since do not have uploadId + // until here + assert.ifError(err); + const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; + const md5Hash = crypto.createHash('md5'); + const bufferBody = Buffer.from(postBody); + const partHash = md5Hash.update(bufferBody).digest('hex'); + const partRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, + query: { + partNumber: '1', + uploadId: testUploadId, + }, + partHash, + }, + postBody, + ); + objectPutPart(authInfo, partRequest, undefined, log, err => { + assert.ifError(err); + const keysInMPUkeyMap = []; + metadata.keyMaps.get(mpuBucket).forEach((val, key) => { + keysInMPUkeyMap.push(key); + }); + const sortedKeyMap = keysInMPUkeyMap.sort(a => { + if (a.slice(0, 8) === 'overview') { + return -1; + } + return 0; + }); + const partKey = sortedKeyMap[1]; + const partETag = metadata.keyMaps.get(mpuBucket).get(partKey)['content-md5']; + assert.strictEqual(keysInMPUkeyMap.length, 2); + assert.strictEqual(partETag, partHash); + done(); + }); }, - partHash, - }, postBody); - objectPutPart(authInfo, partRequest, undefined, log, err => { - assert.ifError(err); - const keysInMPUkeyMap = []; - metadata.keyMaps.get(mpuBucket).forEach((val, key) => { - keysInMPUkeyMap.push(key); - }); - const sortedKeyMap = keysInMPUkeyMap.sort(a => { - if (a.slice(0, 8) === 'overview') { - return -1; - } - return 0; - }); - const partKey = sortedKeyMap[1]; - const partETag = metadata.keyMaps.get(mpuBucket) - .get(partKey)['content-md5']; - assert.strictEqual(keysInMPUkeyMap.length, 2); - assert.strictEqual(partETag, partHash); - done(); - }); - }); - }); + ); + }, + ); it('should return an error if too many parts', done => { - async.waterfall([ - next => bucketPut(authInfo, bucketPutRequest, log, next), - (corsHeaders, next) => initiateMultipartUpload(authInfo, - initiateRequest, log, next), - (result, corsHeaders, next) => parseString(result, next), - ], - (err, json) => { - // Need to build request in here since do not have uploadId - // until here - assert.ifError(err); - const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; - const md5Hash = crypto.createHash('md5'); - const bufferBody = Buffer.from(postBody); - md5Hash.update(bufferBody); - const partHash = md5Hash.digest('hex'); - const partRequest = new DummyRequest({ - bucketName, - namespace, - objectKey, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, - query: { - partNumber: '10001', - uploadId: testUploadId, - }, - partHash, - }, postBody); - objectPutPart(authInfo, partRequest, undefined, log, - (err, result) => { + async.waterfall( + [ + next => bucketPut(authInfo, bucketPutRequest, log, next), + (corsHeaders, next) => initiateMultipartUpload(authInfo, initiateRequest, log, next), + (result, corsHeaders, next) => parseString(result, next), + ], + (err, json) => { + // Need to build request in here since do not have uploadId + // until here + assert.ifError(err); + const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; + const md5Hash = crypto.createHash('md5'); + const bufferBody = Buffer.from(postBody); + md5Hash.update(bufferBody); + const partHash = md5Hash.digest('hex'); + const partRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, + query: { + partNumber: '10001', + uploadId: testUploadId, + }, + partHash, + }, + postBody, + ); + objectPutPart(authInfo, partRequest, undefined, log, (err, result) => { assert(err.is.TooManyParts); assert.strictEqual(result, undefined); done(); }); - }); + }, + ); }); it('should return an error if part number is not an integer', done => { - async.waterfall([ - next => bucketPut(authInfo, bucketPutRequest, log, next), - (corsHeaders, next) => initiateMultipartUpload(authInfo, - initiateRequest, log, next), - (result, corsHeaders, next) => parseString(result, next), - ], - (err, json) => { - // Need to build request in here since do not have uploadId - // until here - assert.ifError(err); - const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; - const md5Hash = crypto.createHash('md5'); - const bufferBody = Buffer.from(postBody); - md5Hash.update(bufferBody); - const partHash = md5Hash.digest('hex'); - const partRequest = new DummyRequest({ - bucketName, - objectKey, - namespace, - url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - query: { - partNumber: 'I am not an integer', - uploadId: testUploadId, - }, - partHash, - }, postBody); - objectPutPart(authInfo, partRequest, undefined, log, - (err, result) => { + async.waterfall( + [ + next => bucketPut(authInfo, bucketPutRequest, log, next), + (corsHeaders, next) => initiateMultipartUpload(authInfo, initiateRequest, log, next), + (result, corsHeaders, next) => parseString(result, next), + ], + (err, json) => { + // Need to build request in here since do not have uploadId + // until here + assert.ifError(err); + const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; + const md5Hash = crypto.createHash('md5'); + const bufferBody = Buffer.from(postBody); + md5Hash.update(bufferBody); + const partHash = md5Hash.digest('hex'); + const partRequest = new DummyRequest( + { + bucketName, + objectKey, + namespace, + url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + query: { + partNumber: 'I am not an integer', + uploadId: testUploadId, + }, + partHash, + }, + postBody, + ); + objectPutPart(authInfo, partRequest, undefined, log, (err, result) => { assert(err.is.InvalidArgument); assert.strictEqual(result, undefined); done(); }); - }); + }, + ); }); it('should return an error if content-length is too large', done => { // Note this is only faking a large file // by setting a large content-length. It is not actually putting a // large file. Functional tests will test actual large data. - async.waterfall([ - next => bucketPut(authInfo, bucketPutRequest, log, next), - (corsHeaders, next) => initiateMultipartUpload(authInfo, - initiateRequest, log, next), - (result, corsHeaders, next) => parseString(result, next), - ], - (err, json) => { - // Need to build request in here since do not have uploadId - // until here - assert.ifError(err); - const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; - const md5Hash = crypto.createHash('md5'); - const bufferBody = Buffer.from(postBody); - md5Hash.update(bufferBody); - const partHash = md5Hash.digest('hex'); - const partRequest = new DummyRequest({ - bucketName, - namespace, - objectKey, - headers: { - 'host': `${bucketName}.s3.amazonaws.com`, - 'content-length': '5368709121', - }, - url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, - query: { - partNumber: '1', - uploadId: testUploadId, - }, - partHash, - parsedContentLength: 5368709121, - }, postBody); - objectPutPart(authInfo, partRequest, undefined, - log, (err, result) => { + async.waterfall( + [ + next => bucketPut(authInfo, bucketPutRequest, log, next), + (corsHeaders, next) => initiateMultipartUpload(authInfo, initiateRequest, log, next), + (result, corsHeaders, next) => parseString(result, next), + ], + (err, json) => { + // Need to build request in here since do not have uploadId + // until here + assert.ifError(err); + const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; + const md5Hash = crypto.createHash('md5'); + const bufferBody = Buffer.from(postBody); + md5Hash.update(bufferBody); + const partHash = md5Hash.digest('hex'); + const partRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey, + headers: { + host: `${bucketName}.s3.amazonaws.com`, + 'content-length': '5368709121', + }, + url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, + query: { + partNumber: '1', + uploadId: testUploadId, + }, + partHash, + parsedContentLength: 5368709121, + }, + postBody, + ); + objectPutPart(authInfo, partRequest, undefined, log, (err, result) => { assert(err.is.EntityTooLarge); assert.strictEqual(result, undefined); done(); }); - }); + }, + ); }); it('should upload two parts', done => { - async.waterfall([ - next => bucketPut(authInfo, bucketPutRequest, log, next), - (corsHeaders, next) => initiateMultipartUpload(authInfo, - initiateRequest, log, next), - (result, corsHeaders, next) => parseString(result, next), - ], - (err, json) => { - // Need to build request in here since do not have uploadId - // until here - assert.ifError(err); - const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; - const md5Hash = crypto.createHash('md5'); - const bufferBody = Buffer.from(postBody); - md5Hash.update(bufferBody); - const partHash = md5Hash.digest('hex'); - const partRequest1 = new DummyRequest({ - bucketName, - namespace, - objectKey, - url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - query: { - partNumber: '1', - uploadId: testUploadId, - }, - partHash, - }, postBody); - objectPutPart(authInfo, partRequest1, undefined, log, () => { - const postBody2 = Buffer.from('I am a second part', 'utf8'); - const md5Hash2 = crypto.createHash('md5'); - const bufferBody2 = Buffer.from(postBody2); - md5Hash2.update(bufferBody2); - const secondCalculatedMD5 = md5Hash2.digest('hex'); - const partRequest2 = new DummyRequest({ - bucketName, - namespace, - objectKey, - url: `/${objectKey}?partNumber=` + - `1&uploadId=${testUploadId}`, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - query: { - partNumber: '2', - uploadId: testUploadId, + async.waterfall( + [ + next => bucketPut(authInfo, bucketPutRequest, log, next), + (corsHeaders, next) => initiateMultipartUpload(authInfo, initiateRequest, log, next), + (result, corsHeaders, next) => parseString(result, next), + ], + (err, json) => { + // Need to build request in here since do not have uploadId + // until here + assert.ifError(err); + const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; + const md5Hash = crypto.createHash('md5'); + const bufferBody = Buffer.from(postBody); + md5Hash.update(bufferBody); + const partHash = md5Hash.digest('hex'); + const partRequest1 = new DummyRequest( + { + bucketName, + namespace, + objectKey, + url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + query: { + partNumber: '1', + uploadId: testUploadId, + }, + partHash, }, - partHash: secondCalculatedMD5, - }, postBody2); - objectPutPart(authInfo, partRequest2, undefined, log, err => { - assert.ifError(err); + postBody, + ); + objectPutPart(authInfo, partRequest1, undefined, log, () => { + const postBody2 = Buffer.from('I am a second part', 'utf8'); + const md5Hash2 = crypto.createHash('md5'); + const bufferBody2 = Buffer.from(postBody2); + md5Hash2.update(bufferBody2); + const secondCalculatedMD5 = md5Hash2.digest('hex'); + const partRequest2 = new DummyRequest( + { + bucketName, + namespace, + objectKey, + url: `/${objectKey}?partNumber=` + `1&uploadId=${testUploadId}`, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + query: { + partNumber: '2', + uploadId: testUploadId, + }, + partHash: secondCalculatedMD5, + }, + postBody2, + ); + objectPutPart(authInfo, partRequest2, undefined, log, err => { + assert.ifError(err); - const keysInMPUkeyMap = []; - metadata.keyMaps.get(mpuBucket).forEach((val, key) => { - keysInMPUkeyMap.push(key); - }); - const sortedKeyMap = keysInMPUkeyMap.sort(a => { - if (a.slice(0, 8) === 'overview') { - return -1; - } - return 0; + const keysInMPUkeyMap = []; + metadata.keyMaps.get(mpuBucket).forEach((val, key) => { + keysInMPUkeyMap.push(key); + }); + const sortedKeyMap = keysInMPUkeyMap.sort(a => { + if (a.slice(0, 8) === 'overview') { + return -1; + } + return 0; + }); + const overviewEntry = sortedKeyMap[0]; + const partKey = sortedKeyMap[2]; + const secondPartEntryArray = partKey.split(splitter); + const partUploadId = secondPartEntryArray[0]; + const secondPartETag = metadata.keyMaps.get(mpuBucket).get(partKey)['content-md5']; + const secondPartNumber = secondPartEntryArray[1]; + assert.strictEqual(keysInMPUkeyMap.length, 3); + assert.strictEqual(metadata.keyMaps.get(mpuBucket).get(overviewEntry).key, objectKey); + assert.strictEqual(partUploadId, testUploadId); + assert.strictEqual(secondPartNumber, '00002'); + assert.strictEqual(secondPartETag, secondCalculatedMD5); + done(); }); - const overviewEntry = sortedKeyMap[0]; - const partKey = sortedKeyMap[2]; - const secondPartEntryArray = partKey.split(splitter); - const partUploadId = secondPartEntryArray[0]; - const secondPartETag = metadata.keyMaps.get(mpuBucket) - .get(partKey)['content-md5']; - const secondPartNumber = secondPartEntryArray[1]; - assert.strictEqual(keysInMPUkeyMap.length, 3); - assert.strictEqual(metadata - .keyMaps.get(mpuBucket).get(overviewEntry).key, - objectKey); - assert.strictEqual(partUploadId, testUploadId); - assert.strictEqual(secondPartNumber, '00002'); - assert.strictEqual(secondPartETag, secondCalculatedMD5); - done(); }); - }); - }); + }, + ); }); it('should complete a multipart upload', done => { const partBody = Buffer.from('I am a part\n', 'utf8'); - initiateRequest.headers['x-amz-meta-stuff'] = - 'I am some user metadata'; - async.waterfall([ - next => bucketPut(authInfo, bucketPutRequest, log, next), - (corsHeaders, next) => initiateMultipartUpload(authInfo, - initiateRequest, log, next), - (result, corsHeaders, next) => parseString(result, next), - ], - (err, json) => { - // Need to build request in here since do not have uploadId - // until here - assert.ifError(err); - const testUploadId = - json.InitiateMultipartUploadResult.UploadId[0]; - const partHash = crypto.createHash('md5').update(partBody).digest('hex'); - const partRequest = new DummyRequest({ - bucketName, - namespace, - objectKey, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, - query: { - partNumber: '1', - uploadId: testUploadId, - }, - // Note that the body of the post set in the request here does - // not really matter in this test. - // The put is not going through the route so the md5 is being - // calculated above and manually being set in the request below. - // What is being tested is that the partHash being sent - // to the API for the part is stored and then used to - // calculate the final ETag upon completion - // of the multipart upload. - partHash, - }, partBody); - objectPutPart(authInfo, partRequest, undefined, log, () => { - const completeBody = '' + - '' + - '1' + - `"${partHash}"` + - '' + - ''; - const completeRequest = { - bucketName, - namespace, - objectKey, - parsedHost: 's3.amazonaws.com', - url: `/${objectKey}?uploadId=${testUploadId}`, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - query: { uploadId: testUploadId }, - post: completeBody, - actionImplicitDenies: false, - }; - const awsVerifiedETag = - '"953e9e776f285afc0bfcf1ab4668299d-1"'; - completeMultipartUpload(authInfo, - completeRequest, log, (err, result) => { + initiateRequest.headers['x-amz-meta-stuff'] = 'I am some user metadata'; + async.waterfall( + [ + next => bucketPut(authInfo, bucketPutRequest, log, next), + (corsHeaders, next) => initiateMultipartUpload(authInfo, initiateRequest, log, next), + (result, corsHeaders, next) => parseString(result, next), + ], + (err, json) => { + // Need to build request in here since do not have uploadId + // until here + assert.ifError(err); + const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; + const partHash = crypto.createHash('md5').update(partBody).digest('hex'); + const partRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, + query: { + partNumber: '1', + uploadId: testUploadId, + }, + // Note that the body of the post set in the request here does + // not really matter in this test. + // The put is not going through the route so the md5 is being + // calculated above and manually being set in the request below. + // What is being tested is that the partHash being sent + // to the API for the part is stored and then used to + // calculate the final ETag upon completion + // of the multipart upload. + partHash, + }, + partBody, + ); + objectPutPart(authInfo, partRequest, undefined, log, () => { + const completeBody = + '' + + '' + + '1' + + `"${partHash}"` + + '' + + ''; + const completeRequest = { + bucketName, + namespace, + objectKey, + parsedHost: 's3.amazonaws.com', + url: `/${objectKey}?uploadId=${testUploadId}`, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + query: { uploadId: testUploadId }, + post: completeBody, + actionImplicitDenies: false, + }; + const awsVerifiedETag = '"953e9e776f285afc0bfcf1ab4668299d-1"'; + completeMultipartUpload(authInfo, completeRequest, log, (err, result) => { assert.ifError(err); parseString(result, (err, json) => { assert.ifError(err); assert.strictEqual( json.CompleteMultipartUploadResult.Location[0], - `http://${bucketName}.s3.amazonaws.com` - + `/${objectKey}`); - assert.strictEqual( - json.CompleteMultipartUploadResult.Bucket[0], - bucketName); - assert.strictEqual( - json.CompleteMultipartUploadResult.Key[0], - objectKey); - assert.strictEqual( - json.CompleteMultipartUploadResult.ETag[0], - awsVerifiedETag); - const MD = metadata.keyMaps.get(bucketName) - .get(objectKey); + `http://${bucketName}.s3.amazonaws.com` + `/${objectKey}`, + ); + assert.strictEqual(json.CompleteMultipartUploadResult.Bucket[0], bucketName); + assert.strictEqual(json.CompleteMultipartUploadResult.Key[0], objectKey); + assert.strictEqual(json.CompleteMultipartUploadResult.ETag[0], awsVerifiedETag); + const MD = metadata.keyMaps.get(bucketName).get(objectKey); assert(MD); - assert.strictEqual(MD['x-amz-meta-stuff'], - 'I am some user metadata'); + assert.strictEqual(MD['x-amz-meta-stuff'], 'I am some user metadata'); assert.strictEqual(MD.uploadId, testUploadId); done(); }); }); - }); - }); - }); - - it('should complete a multipart upload even if etag is sent ' + - 'in post body without quotes (a la Cyberduck)', done => { - const partBody = Buffer.from('I am a part\n', 'utf8'); - initiateRequest.headers['x-amz-meta-stuff'] = - 'I am some user metadata'; - async.waterfall([ - function waterfall1(next) { - bucketPut(authInfo, bucketPutRequest, log, next); - }, - function waterfall2(corsHeaders, next) { - initiateMultipartUpload( - authInfo, initiateRequest, log, next); - }, - function waterfall3(result, corsHeaders, next) { - parseString(result, next); + }); }, - ], - (err, json) => { - // Need to build request in here since do not have uploadId - // until here - assert.ifError(err); - const testUploadId = - json.InitiateMultipartUploadResult.UploadId[0]; - const partHash = crypto.createHash('md5').update(partBody).digest('hex'); - const partRequest = new DummyRequest({ - bucketName, - namespace, - objectKey, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, - query: { - partNumber: '1', - uploadId: testUploadId, - }, - partHash, - }, partBody); - objectPutPart(authInfo, partRequest, undefined, log, () => { - const completeBody = '' + - '' + - '1' + - // ETag without quotes - `${partHash}` + - '' + - ''; - const completeRequest = { - bucketName, - namespace, - objectKey, - parsedHost: 's3.amazonaws.com', - url: `/${objectKey}?uploadId=${testUploadId}`, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - query: { uploadId: testUploadId }, - post: completeBody, - actionImplicitDenies: false, - }; - const awsVerifiedETag = - '"953e9e776f285afc0bfcf1ab4668299d-1"'; - completeMultipartUpload(authInfo, - completeRequest, log, (err, result) => { - assert.ifError(err); - parseString(result, (err, json) => { - assert.ifError(err); - assert.strictEqual( - json.CompleteMultipartUploadResult.Location[0], - `http://${bucketName}.s3.amazonaws.com` - + `/${objectKey}`); - assert.strictEqual( - json.CompleteMultipartUploadResult.Bucket[0], - bucketName); - assert.strictEqual( - json.CompleteMultipartUploadResult.Key[0], - objectKey); - assert.strictEqual( - json.CompleteMultipartUploadResult.ETag[0], - awsVerifiedETag); - const MD = metadata.keyMaps.get(bucketName) - .get(objectKey); - assert(MD); - assert.strictEqual(MD['x-amz-meta-stuff'], - 'I am some user metadata'); - done(); - }); - }); - }); - }); + ); }); - it('should return an error if a complete multipart upload' + - ' request contains malformed xml', done => { - async.waterfall([ - next => bucketPut(authInfo, bucketPutRequest, log, next), - (corsHeaders, next) => initiateMultipartUpload(authInfo, - initiateRequest, log, next), - (result, corsHeaders, next) => parseString(result, next), - ], - (err, json) => { - // Need to build request in here since do not have uploadId - // until here - const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; - const md5Hash = crypto.createHash('md5'); - const bufferBody = Buffer.from(postBody); - md5Hash.update(bufferBody); - const partHash = md5Hash.digest('hex'); - const partRequest = new DummyRequest({ - bucketName, - namespace, - objectKey, - url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - query: { - partNumber: '1', - uploadId: testUploadId, - }, - partHash, - }, postBody); - objectPutPart(authInfo, partRequest, undefined, log, () => { - const completeBody = 'Malformed xml'; - const completeRequest = { - bucketName, - objectKey, - namespace, - url: `/${objectKey}?uploadId=${testUploadId}`, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - query: { uploadId: testUploadId }, - post: completeBody, - partHash, - actionImplicitDenies: false, - }; - completeMultipartUpload(authInfo, - completeRequest, log, err => { - assert.strictEqual(err.is.MalformedXML, true); - assert.strictEqual(metadata.keyMaps.get(mpuBucket).size, - 2); - done(); + it( + 'should complete a multipart upload even if etag is sent ' + 'in post body without quotes (a la Cyberduck)', + done => { + const partBody = Buffer.from('I am a part\n', 'utf8'); + initiateRequest.headers['x-amz-meta-stuff'] = 'I am some user metadata'; + async.waterfall( + [ + function waterfall1(next) { + bucketPut(authInfo, bucketPutRequest, log, next); + }, + function waterfall2(corsHeaders, next) { + initiateMultipartUpload(authInfo, initiateRequest, log, next); + }, + function waterfall3(result, corsHeaders, next) { + parseString(result, next); + }, + ], + (err, json) => { + // Need to build request in here since do not have uploadId + // until here + assert.ifError(err); + const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; + const partHash = crypto.createHash('md5').update(partBody).digest('hex'); + const partRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, + query: { + partNumber: '1', + uploadId: testUploadId, + }, + partHash, + }, + partBody, + ); + objectPutPart(authInfo, partRequest, undefined, log, () => { + const completeBody = + '' + + '' + + '1' + + // ETag without quotes + `${partHash}` + + '' + + ''; + const completeRequest = { + bucketName, + namespace, + objectKey, + parsedHost: 's3.amazonaws.com', + url: `/${objectKey}?uploadId=${testUploadId}`, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + query: { uploadId: testUploadId }, + post: completeBody, + actionImplicitDenies: false, + }; + const awsVerifiedETag = '"953e9e776f285afc0bfcf1ab4668299d-1"'; + completeMultipartUpload(authInfo, completeRequest, log, (err, result) => { + assert.ifError(err); + parseString(result, (err, json) => { + assert.ifError(err); + assert.strictEqual( + json.CompleteMultipartUploadResult.Location[0], + `http://${bucketName}.s3.amazonaws.com` + `/${objectKey}`, + ); + assert.strictEqual(json.CompleteMultipartUploadResult.Bucket[0], bucketName); + assert.strictEqual(json.CompleteMultipartUploadResult.Key[0], objectKey); + assert.strictEqual(json.CompleteMultipartUploadResult.ETag[0], awsVerifiedETag); + const MD = metadata.keyMaps.get(bucketName).get(objectKey); + assert(MD); + assert.strictEqual(MD['x-amz-meta-stuff'], 'I am some user metadata'); + done(); + }); + }); }); - }); - }); - }); - - it('should return an error if the complete ' + - 'multipart upload request contains xml that ' + - 'does not conform to the AWS spec', done => { - async.waterfall([ - next => bucketPut(authInfo, bucketPutRequest, log, next), - (corsHeaders, next) => initiateMultipartUpload(authInfo, - initiateRequest, log, next), - (result, corsHeaders, next) => parseString(result, next), - ], - (err, json) => { - // Need to build request in here since do not have uploadId - // until here - const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; - const md5Hash = crypto.createHash('md5'); - const bufferBody = Buffer.from(postBody); - md5Hash.update(bufferBody); - const partHash = md5Hash.digest('hex'); - const partRequest = new DummyRequest({ - bucketName, - namespace, - objectKey, - url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - query: { - partNumber: '1', - uploadId: testUploadId, - }, - partHash, - }, postBody); - objectPutPart(authInfo, partRequest, undefined, log, () => { - // XML is missing any part listing so does - // not conform to the AWS spec - const completeBody = '' + - ''; - const completeRequest = { - bucketName, - namespace, - objectKey, - url: `/${objectKey}?uploadId=${testUploadId}`, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - query: { uploadId: testUploadId }, - post: completeBody, - partHash, - actionImplicitDenies: false, - }; - completeMultipartUpload(authInfo, completeRequest, log, err => { - assert(err.is.MalformedXML); - done(); - }); - }); - }); - }); - - it('should return an error if the complete ' + - 'multipart upload request contains xml with ' + - 'a part list that is not in numerical order', done => { - async.waterfall([ - next => bucketPut(authInfo, bucketPutRequest, log, next), - (corsHeaders, next) => initiateMultipartUpload(authInfo, - initiateRequest, log, next), - (result, corsHeaders, next) => parseString(result, next), - ], - (err, json) => { - // Need to build request in here since do not have uploadId - // until here - const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; - const md5Hash = crypto.createHash('md5'); - const fullSizedPart = crypto.randomBytes(5 * 1024 * 1024); - const bufferBody = Buffer.from(fullSizedPart); - md5Hash.update(bufferBody); - const partHash = md5Hash.digest('hex'); - const partRequest1 = new DummyRequest({ - bucketName, - namespace, - objectKey, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, - query: { - partNumber: '1', - uploadId: testUploadId, }, - partHash, - }, fullSizedPart); - const partRequest2 = new DummyRequest({ - bucketName, - namespace, - objectKey, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, - query: { - partNumber: '2', - uploadId: testUploadId, - }, - partHash, - }, fullSizedPart); - objectPutPart(authInfo, partRequest1, undefined, log, () => { - objectPutPart(authInfo, partRequest2, undefined, log, () => { - const completeBody = '' + - '' + - '2' + - `"${partHash}"` + - '' + - '' + - '1' + - `"${partHash}"` + - '' + - ''; - const completeRequest = { + ); + }, + ); + + it('should return an error if a complete multipart upload' + ' request contains malformed xml', done => { + async.waterfall( + [ + next => bucketPut(authInfo, bucketPutRequest, log, next), + (corsHeaders, next) => initiateMultipartUpload(authInfo, initiateRequest, log, next), + (result, corsHeaders, next) => parseString(result, next), + ], + (err, json) => { + // Need to build request in here since do not have uploadId + // until here + const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; + const md5Hash = crypto.createHash('md5'); + const bufferBody = Buffer.from(postBody); + md5Hash.update(bufferBody); + const partHash = md5Hash.digest('hex'); + const partRequest = new DummyRequest( + { bucketName, namespace, objectKey, + url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + query: { + partNumber: '1', + uploadId: testUploadId, + }, + partHash, + }, + postBody, + ); + objectPutPart(authInfo, partRequest, undefined, log, () => { + const completeBody = 'Malformed xml'; + const completeRequest = { + bucketName, + objectKey, + namespace, url: `/${objectKey}?uploadId=${testUploadId}`, headers: { host: `${bucketName}.s3.amazonaws.com` }, query: { uploadId: testUploadId }, @@ -907,1115 +775,1311 @@ describe('Multipart Upload API', () => { partHash, actionImplicitDenies: false, }; - completeMultipartUpload(authInfo, - completeRequest, log, err => { - assert(err.is.InvalidPartOrder); - assert.strictEqual(metadata.keyMaps - .get(mpuBucket).size, 3); + completeMultipartUpload(authInfo, completeRequest, log, err => { + assert.strictEqual(err.is.MalformedXML, true); + assert.strictEqual(metadata.keyMaps.get(mpuBucket).size, 2); + done(); + }); + }); + }, + ); + }); + + it( + 'should return an error if the complete ' + + 'multipart upload request contains xml that ' + + 'does not conform to the AWS spec', + done => { + async.waterfall( + [ + next => bucketPut(authInfo, bucketPutRequest, log, next), + (corsHeaders, next) => initiateMultipartUpload(authInfo, initiateRequest, log, next), + (result, corsHeaders, next) => parseString(result, next), + ], + (err, json) => { + // Need to build request in here since do not have uploadId + // until here + const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; + const md5Hash = crypto.createHash('md5'); + const bufferBody = Buffer.from(postBody); + md5Hash.update(bufferBody); + const partHash = md5Hash.digest('hex'); + const partRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey, + url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + query: { + partNumber: '1', + uploadId: testUploadId, + }, + partHash, + }, + postBody, + ); + objectPutPart(authInfo, partRequest, undefined, log, () => { + // XML is missing any part listing so does + // not conform to the AWS spec + const completeBody = '' + ''; + const completeRequest = { + bucketName, + namespace, + objectKey, + url: `/${objectKey}?uploadId=${testUploadId}`, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + query: { uploadId: testUploadId }, + post: completeBody, + partHash, + actionImplicitDenies: false, + }; + completeMultipartUpload(authInfo, completeRequest, log, err => { + assert(err.is.MalformedXML); + done(); + }); + }); + }, + ); + }, + ); + + it( + 'should return an error if the complete ' + + 'multipart upload request contains xml with ' + + 'a part list that is not in numerical order', + done => { + async.waterfall( + [ + next => bucketPut(authInfo, bucketPutRequest, log, next), + (corsHeaders, next) => initiateMultipartUpload(authInfo, initiateRequest, log, next), + (result, corsHeaders, next) => parseString(result, next), + ], + (err, json) => { + // Need to build request in here since do not have uploadId + // until here + const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; + const md5Hash = crypto.createHash('md5'); + const fullSizedPart = crypto.randomBytes(5 * 1024 * 1024); + const bufferBody = Buffer.from(fullSizedPart); + md5Hash.update(bufferBody); + const partHash = md5Hash.digest('hex'); + const partRequest1 = new DummyRequest( + { + bucketName, + namespace, + objectKey, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, + query: { + partNumber: '1', + uploadId: testUploadId, + }, + partHash, + }, + fullSizedPart, + ); + const partRequest2 = new DummyRequest( + { + bucketName, + namespace, + objectKey, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, + query: { + partNumber: '2', + uploadId: testUploadId, + }, + partHash, + }, + fullSizedPart, + ); + objectPutPart(authInfo, partRequest1, undefined, log, () => { + objectPutPart(authInfo, partRequest2, undefined, log, () => { + const completeBody = + '' + + '' + + '2' + + `"${partHash}"` + + '' + + '' + + '1' + + `"${partHash}"` + + '' + + ''; + const completeRequest = { + bucketName, + namespace, + objectKey, + url: `/${objectKey}?uploadId=${testUploadId}`, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + query: { uploadId: testUploadId }, + post: completeBody, + partHash, + actionImplicitDenies: false, + }; + completeMultipartUpload(authInfo, completeRequest, log, err => { + assert(err.is.InvalidPartOrder); + assert.strictEqual(metadata.keyMaps.get(mpuBucket).size, 3); + done(); + }); + }); + }); + }, + ); + }, + ); + + it( + 'should return InvalidPart error if the complete ' + + 'multipart upload request contains xml with a missing part', + done => { + async.waterfall( + [ + next => bucketPut(authInfo, bucketPutRequest, log, next), + (corsHeaders, next) => initiateMultipartUpload(authInfo, initiateRequest, log, next), + (result, corsHeaders, next) => parseString(result, next), + ], + (err, json) => { + // Need to build request in here since do not have uploadId + // until here + const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; + const md5Hash = crypto.createHash('md5'); + const fullSizedPart = crypto.randomBytes(5 * 1024 * 1024); + const bufferBody = Buffer.from(fullSizedPart); + md5Hash.update(bufferBody); + const partHash = md5Hash.digest('hex'); + const partRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, + query: { + partNumber: '1', + uploadId: testUploadId, + }, + partHash, + }, + fullSizedPart, + ); + objectPutPart(authInfo, partRequest, undefined, log, () => { + const completeBody = + '' + + '' + + '99999' + + `"${partHash}"` + + '' + + ''; + const completeRequest = { + bucketName, + namespace, + objectKey, + url: `/${objectKey}?uploadId=${testUploadId}`, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + query: { uploadId: testUploadId }, + post: completeBody, + partHash, + actionImplicitDenies: false, + }; + completeMultipartUpload(authInfo, completeRequest, log, err => { + assert(err.is.InvalidPart); + assert.strictEqual(metadata.keyMaps.get(mpuBucket).size, 2); done(); }); + }); + }, + ); + }, + ); + + it( + 'should return an error if the complete multipart upload request ' + + 'contains xml with a part ETag that does not match the md5 for ' + + 'the part that was actually sent', + done => { + async.waterfall( + [ + next => bucketPut(authInfo, bucketPutRequest, log, next), + (corsHeaders, next) => initiateMultipartUpload(authInfo, initiateRequest, log, next), + (result, corsHeaders, next) => parseString(result, next), + ], + (err, json) => { + // Need to build request in here since do not have uploadId + // until here + const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; + const wrongMD5 = '3858f62230ac3c915f300c664312c11f-9'; + const fullSizedPart = crypto.randomBytes(5 * 1024 * 1024); + const partRequest1 = new DummyRequest( + { + bucketName, + namespace, + objectKey, + url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + query: { + partNumber: '1', + uploadId: testUploadId, + }, + }, + fullSizedPart, + ); + const partRequest2 = new DummyRequest( + { + bucketName, + namespace, + objectKey, + url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + query: { + partNumber: '2', + uploadId: testUploadId, + }, + }, + postBody, + ); + objectPutPart(authInfo, partRequest1, undefined, log, err => { + assert.deepStrictEqual(err, null); + const partHash = partRequest1.partHash; + objectPutPart(authInfo, partRequest2, undefined, log, err => { + assert.deepStrictEqual(err, null); + const completeBody = + '' + + '' + + '1' + + `"${partHash}"` + + '' + + '' + + '2' + + `${wrongMD5}` + + '' + + ''; + const completeRequest = { + bucketName, + namespace, + objectKey, + url: `/${objectKey}?uploadId=${testUploadId}`, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + query: { uploadId: testUploadId }, + post: completeBody, + partHash, + actionImplicitDenies: false, + }; + assert.strictEqual(metadata.keyMaps.get(mpuBucket).size, 3); + completeMultipartUpload(authInfo, completeRequest, log, err => { + assert(err.is.InvalidPart); + done(); + }); + }); + }); + }, + ); + }, + ); + + it( + 'should return an error if there is a part ' + 'other than the last part that is less than 5MB ' + 'in size', + done => { + async.waterfall( + [ + next => bucketPut(authInfo, bucketPutRequest, log, next), + (corsHeaders, next) => initiateMultipartUpload(authInfo, initiateRequest, log, next), + (result, corsHeaders, next) => parseString(result, next), + ], + (err, json) => { + // Need to build request in here since do not have uploadId + // until here + const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; + const md5Hash = crypto.createHash('md5'); + const bufferBody = Buffer.from(postBody); + md5Hash.update(bufferBody); + const partHash = md5Hash.digest('hex'); + const partRequest1 = new DummyRequest( + { + bucketName, + namespace, + objectKey, + headers: { + host: `${bucketName}.s3.amazonaws.com`, + 'content-length': '100', + }, + parsedContentLength: 100, + url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, + query: { + partNumber: '1', + uploadId: testUploadId, + }, + partHash, + }, + postBody, + ); + const partRequest2 = new DummyRequest( + { + bucketName, + namespace, + objectKey, + headers: { + host: `${bucketName}.s3.amazonaws.com`, + 'content-length': '200', + }, + parsedContentLength: 200, + url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, + query: { + partNumber: '2', + uploadId: testUploadId, + }, + partHash, + }, + postBody, + ); + objectPutPart(authInfo, partRequest1, undefined, log, () => { + objectPutPart(authInfo, partRequest2, undefined, log, () => { + const completeBody = + '' + + '' + + '1' + + `"${partHash}"` + + '' + + '' + + '2' + + `"${partHash}"` + + '' + + ''; + const completeRequest = { + bucketName, + namespace, + objectKey, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: `/${objectKey}?uploadId=${testUploadId}`, + query: { uploadId: testUploadId }, + post: completeBody, + partHash, + actionImplicitDenies: false, + }; + assert.strictEqual(metadata.keyMaps.get(mpuBucket).size, 3); + completeMultipartUpload(authInfo, completeRequest, log, err => { + assert(err.is.EntityTooSmall); + done(); + }); + }); + }); + }, + ); + }, + ); + + it('should aggregate the sizes of the parts', done => { + async.waterfall( + [ + next => bucketPut(authInfo, bucketPutRequest, log, next), + (corsHeaders, next) => initiateMultipartUpload(authInfo, initiateRequest, log, next), + (result, corsHeaders, next) => parseString(result, next), + ], + (err, json) => { + // Need to build request in here since do not have uploadId + // until her + const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; + const md5Hash = crypto.createHash('md5'); + const bufferBody = Buffer.from(postBody); + md5Hash.update(bufferBody); + const partHash = md5Hash.digest('hex'); + const partRequest1 = new DummyRequest( + { + bucketName, + namespace, + objectKey, + headers: { + host: `${bucketName}.s3.amazonaws.com`, + 'content-length': '6000000', + }, + parsedContentLength: 6000000, + url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, + query: { + partNumber: '1', + uploadId: testUploadId, + }, + partHash, + }, + postBody, + ); + const partRequest2 = new DummyRequest( + { + bucketName, + namespace, + objectKey, + headers: { + host: `${bucketName}.s3.amazonaws.com`, + 'content-length': '100', + }, + parsedContentLength: 100, + url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, + query: { + partNumber: '2', + uploadId: testUploadId, + }, + post: postBody, + partHash, + }, + postBody, + ); + objectPutPart(authInfo, partRequest1, undefined, log, () => { + objectPutPart(authInfo, partRequest2, undefined, log, () => { + const completeBody = + '' + + '' + + '1' + + `"${partHash}"` + + '' + + '' + + '2' + + `"${partHash}"` + + '' + + ''; + const completeRequest = { + bucketName, + namespace, + objectKey, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: `/${objectKey}?uploadId=${testUploadId}`, + query: { uploadId: testUploadId }, + post: completeBody, + partHash, + actionImplicitDenies: false, + }; + completeMultipartUpload(authInfo, completeRequest, log, (err, result) => { + assert.strictEqual(err, null); + parseString(result, err => { + assert.strictEqual(err, null); + const MD = metadata.keyMaps.get(bucketName).get(objectKey); + assert(MD); + assert.strictEqual(MD['content-length'], 6000100); + done(); + }); + }); + }); }); - }); - }); + }, + ); }); - it('should return InvalidPart error if the complete ' + - 'multipart upload request contains xml with a missing part', done => { - async.waterfall([ - next => bucketPut(authInfo, bucketPutRequest, log, next), - (corsHeaders, next) => initiateMultipartUpload(authInfo, - initiateRequest, log, next), - (result, corsHeaders, next) => parseString(result, next), - ], - (err, json) => { - // Need to build request in here since do not have uploadId - // until here - const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; - const md5Hash = crypto.createHash('md5'); - const fullSizedPart = crypto.randomBytes(5 * 1024 * 1024); - const bufferBody = Buffer.from(fullSizedPart); - md5Hash.update(bufferBody); - const partHash = md5Hash.digest('hex'); - const partRequest = new DummyRequest({ - bucketName, - namespace, - objectKey, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, - query: { - partNumber: '1', - uploadId: testUploadId, - }, - partHash, - }, fullSizedPart); - objectPutPart(authInfo, partRequest, undefined, log, () => { - const completeBody = '' + - '' + - '99999' + - `"${partHash}"` + - '' + - ''; - const completeRequest = { - bucketName, - namespace, - objectKey, - url: `/${objectKey}?uploadId=${testUploadId}`, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - query: { uploadId: testUploadId }, - post: completeBody, - partHash, - actionImplicitDenies: false, - }; - completeMultipartUpload(authInfo, completeRequest, log, err => { - assert(err.is.InvalidPart); - assert.strictEqual(metadata.keyMaps.get(mpuBucket).size, 2); - done(); + it('should set a canned ACL for a multipart upload', done => { + const initiateRequest = { + bucketName, + namespace, + objectKey, + headers: { + host: `${bucketName}.s3.amazonaws.com`, + 'x-amz-meta-stuff': 'I am some user metadata', + 'x-amz-acl': 'authenticated-read', + }, + url: `/${objectKey}?uploads`, + actionImplicitDenies: false, + }; + + async.waterfall( + [ + next => bucketPut(authInfo, bucketPutRequest, log, next), + (corsHeaders, next) => initiateMultipartUpload(authInfo, initiateRequest, log, next), + (result, corsHeaders, next) => parseString(result, next), + ], + (err, json) => { + // Need to build request in here since do not have uploadId + // until here + const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; + const md5Hash = crypto.createHash('md5'); + const bufferBody = Buffer.from(postBody); + md5Hash.update(bufferBody); + const partHash = md5Hash.digest('hex'); + const partRequest1 = new DummyRequest( + { + bucketName, + namespace, + objectKey, + headers: { + host: `${bucketName}.s3.amazonaws.com`, + 'content-length': 6000000, + }, + url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, + query: { + partNumber: '1', + uploadId: testUploadId, + }, + partHash, + }, + postBody, + ); + const partRequest2 = new DummyRequest( + { + bucketName, + namespace, + objectKey, + headers: { + host: `${bucketName}.s3.amazonaws.com`, + 'content-length': 100, + }, + url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, + query: { + partNumber: '2', + uploadId: testUploadId, + }, + partHash, + }, + postBody, + ); + objectPutPart(authInfo, partRequest1, undefined, log, () => { + objectPutPart(authInfo, partRequest2, undefined, log, () => { + const completeBody = + '' + + '' + + '1' + + `"${partHash}"` + + '' + + '' + + '2' + + `"${partHash}"` + + '' + + ''; + const completeRequest = { + bucketName, + namespace, + objectKey, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: `/${objectKey}?uploadId=${testUploadId}`, + query: { uploadId: testUploadId }, + post: completeBody, + partHash, + actionImplicitDenies: false, + }; + completeMultipartUpload(authInfo, completeRequest, log, (err, result) => { + assert.strictEqual(err, null); + parseString(result, err => { + assert.strictEqual(err, null); + const MD = metadata.keyMaps.get(bucketName).get(objectKey); + assert(MD); + assert.strictEqual(MD.acl.Canned, 'authenticated-read'); + done(); + }); + }); + }); }); - }); - }); + }, + ); }); - it('should return an error if the complete multipart upload request ' - + 'contains xml with a part ETag that does not match the md5 for ' - + 'the part that was actually sent', done => { - async.waterfall([ - next => bucketPut(authInfo, bucketPutRequest, log, next), - (corsHeaders, next) => initiateMultipartUpload(authInfo, - initiateRequest, log, next), - (result, corsHeaders, next) => parseString(result, next), - ], - (err, json) => { - // Need to build request in here since do not have uploadId - // until here - const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; - const wrongMD5 = '3858f62230ac3c915f300c664312c11f-9'; - const fullSizedPart = crypto.randomBytes(5 * 1024 * 1024); - const partRequest1 = new DummyRequest({ - bucketName, - namespace, - objectKey, - url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - query: { - partNumber: '1', - uploadId: testUploadId, - }, - }, fullSizedPart); - const partRequest2 = new DummyRequest({ - bucketName, - namespace, - objectKey, - url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - query: { - partNumber: '2', - uploadId: testUploadId, - }, - }, postBody); - objectPutPart(authInfo, partRequest1, undefined, log, err => { - assert.deepStrictEqual(err, null); - const partHash = partRequest1.partHash; - objectPutPart(authInfo, partRequest2, undefined, log, err => { - assert.deepStrictEqual(err, null); - const completeBody = '' + - '' + - '1' + - `"${partHash}"` + - '' + - '' + - '2' + - `${wrongMD5}` + - '' + - ''; - const completeRequest = { + it('should set specific ACL grants for a multipart upload', done => { + const granteeId = '79a59df900b949e55d96a1e698fbace' + 'dfd6e09d98eacf8f8d5218e7cd47ef2be'; + const granteeEmail = 'sampleAccount1@sampling.com'; + const initiateRequest = { + bucketName, + namespace, + objectKey, + headers: { + host: `${bucketName}.s3.amazonaws.com`, + 'x-amz-meta-stuff': 'I am some user metadata', + 'x-amz-grant-read': `emailAddress="${granteeEmail}"`, + }, + url: `/${objectKey}?uploads`, + actionImplicitDenies: false, + }; + + async.waterfall( + [ + next => bucketPut(authInfo, bucketPutRequest, log, next), + (corsHeaders, next) => initiateMultipartUpload(authInfo, initiateRequest, log, next), + (result, corsHeaders, next) => parseString(result, next), + ], + (err, json) => { + // Need to build request in here since do not have uploadId + // until here + const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; + const md5Hash = crypto.createHash('md5'); + const bufferBody = Buffer.from(postBody); + md5Hash.update(bufferBody); + const partHash = md5Hash.digest('hex'); + const partRequest1 = new DummyRequest( + { bucketName, namespace, objectKey, - url: `/${objectKey}?uploadId=${testUploadId}`, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - query: { uploadId: testUploadId }, - post: completeBody, + headers: { + host: `${bucketName}.s3.amazonaws.com`, + 'content-length': 6000000, + }, + url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, + query: { + partNumber: '1', + uploadId: testUploadId, + }, partHash, - actionImplicitDenies: false, - }; - assert.strictEqual(metadata.keyMaps.get(mpuBucket).size, 3); - completeMultipartUpload(authInfo, - completeRequest, log, err => { - assert(err.is.InvalidPart); - done(); + }, + postBody, + ); + const partRequest2 = new DummyRequest( + { + bucketName, + namespace, + objectKey, + headers: { + host: `${bucketName}.s3.amazonaws.com`, + 'content-length': 100, + }, + url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, + query: { + partNumber: '2', + uploadId: testUploadId, + }, + post: postBody, + partHash, + }, + postBody, + ); + objectPutPart(authInfo, partRequest1, undefined, log, () => { + objectPutPart(authInfo, partRequest2, undefined, log, () => { + const completeBody = + '' + + '' + + '1' + + `"${partHash}"` + + '' + + '' + + '2' + + `"${partHash}"` + + '' + + ''; + const completeRequest = { + bucketName, + namespace, + objectKey, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: `/${objectKey}?uploadId=${testUploadId}`, + query: { uploadId: testUploadId }, + post: completeBody, + partHash, + actionImplicitDenies: false, + }; + completeMultipartUpload(authInfo, completeRequest, log, (err, result) => { + assert.strictEqual(err, null); + parseString(result, err => { + assert.strictEqual(err, null); + const MD = metadata.keyMaps.get(bucketName).get(objectKey); + assert(MD); + assert.strictEqual(MD.acl.READ[0], granteeId); + done(); + }); }); + }); }); - }); - }); + }, + ); }); - it('should return an error if there is a part ' + - 'other than the last part that is less than 5MB ' + - 'in size', done => { - async.waterfall([ - next => bucketPut(authInfo, bucketPutRequest, log, next), - (corsHeaders, next) => initiateMultipartUpload(authInfo, - initiateRequest, log, next), - (result, corsHeaders, next) => parseString(result, next), - ], - (err, json) => { - // Need to build request in here since do not have uploadId - // until here - const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; - const md5Hash = crypto.createHash('md5'); - const bufferBody = Buffer.from(postBody); - md5Hash.update(bufferBody); - const partHash = md5Hash.digest('hex'); - const partRequest1 = new DummyRequest({ - bucketName, - namespace, - objectKey, - headers: { - 'host': `${bucketName}.s3.amazonaws.com`, - 'content-length': '100', - }, - parsedContentLength: 100, - url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, - query: { - partNumber: '1', - uploadId: testUploadId, - }, - partHash, - }, postBody); - const partRequest2 = new DummyRequest({ - bucketName, - namespace, - objectKey, - headers: { - 'host': `${bucketName}.s3.amazonaws.com`, - 'content-length': '200', - }, - parsedContentLength: 200, - url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, - query: { - partNumber: '2', - uploadId: testUploadId, - }, - partHash, - }, postBody); - objectPutPart(authInfo, partRequest1, undefined, log, () => { - objectPutPart(authInfo, partRequest2, undefined, log, () => { - const completeBody = '' + - '' + - '1' + - `"${partHash}"` + - '' + - '' + - '2' + - `"${partHash}"` + - '' + - ''; - const completeRequest = { + it('should abort/delete a multipart upload', done => { + async.waterfall( + [ + next => bucketPut(authInfo, bucketPutRequest, log, next), + (corsHeaders, next) => initiateMultipartUpload(authInfo, initiateRequest, log, next), + (result, corsHeaders, next) => parseString(result, next), + ], + (err, json) => { + // Need to build request in here since do not have uploadId + // until here + const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; + const bufferMD5 = Buffer.from(postBody, 'base64'); + const partHash = bufferMD5.toString('hex'); + const partRequest = new DummyRequest( + { bucketName, namespace, objectKey, + url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, headers: { host: `${bucketName}.s3.amazonaws.com` }, + query: { + partNumber: '1', + uploadId: testUploadId, + }, + partHash, + }, + postBody, + ); + objectPutPart(authInfo, partRequest, undefined, log, () => { + const deleteRequest = { + bucketName, + namespace, + objectKey, url: `/${objectKey}?uploadId=${testUploadId}`, + headers: { host: `${bucketName}.s3.amazonaws.com` }, query: { uploadId: testUploadId }, - post: completeBody, - partHash, actionImplicitDenies: false, }; - assert.strictEqual(metadata.keyMaps.get(mpuBucket).size, 3); - completeMultipartUpload(authInfo, - completeRequest, log, err => { - assert(err.is.EntityTooSmall); - done(); - }); + assert.strictEqual(metadata.keyMaps.get(mpuBucket).size, 2); + multipartDelete(authInfo, deleteRequest, log, err => { + assert.strictEqual(err, null); + assert.strictEqual(metadata.keyMaps.get(mpuBucket).size, 0); + done(); + }); }); - }); - }); + }, + ); }); - it('should aggregate the sizes of the parts', done => { - async.waterfall([ - next => bucketPut(authInfo, bucketPutRequest, log, next), - (corsHeaders, next) => initiateMultipartUpload(authInfo, - initiateRequest, log, next), - (result, corsHeaders, next) => parseString(result, next), - ], - (err, json) => { - // Need to build request in here since do not have uploadId - // until her - const testUploadId = - json.InitiateMultipartUploadResult.UploadId[0]; - const md5Hash = crypto.createHash('md5'); - const bufferBody = Buffer.from(postBody); - md5Hash.update(bufferBody); - const partHash = md5Hash.digest('hex'); - const partRequest1 = new DummyRequest({ - bucketName, - namespace, - objectKey, - headers: { - 'host': `${bucketName}.s3.amazonaws.com`, - 'content-length': '6000000', - }, - parsedContentLength: 6000000, - url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, - query: { - partNumber: '1', - uploadId: testUploadId, + it( + 'should return no error if attempt to abort/delete ' + + 'a multipart upload that does not exist and not using ' + + 'legacyAWSBehavior', + done => { + async.waterfall( + [ + next => bucketPut(authInfo, bucketPutRequest, log, next), + (corsHeaders, next) => initiateMultipartUpload(authInfo, initiateRequest, log, next), + (result, corsHeaders, next) => { + const mpuKeys = metadata.keyMaps.get(mpuBucket); + assert.strictEqual(mpuKeys.size, 1); + parseString(result, next); + }, + ], + (err, json) => { + // Need to build request in here since do not have uploadId + // until here + const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; + const bufferMD5 = Buffer.from(postBody, 'base64'); + const partHash = bufferMD5.toString('hex'); + const partRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, + query: { + partNumber: '1', + uploadId: testUploadId, + }, + partHash, + }, + postBody, + ); + objectPutPart(authInfo, partRequest, undefined, log, () => { + const deleteRequest = { + bucketName, + namespace, + objectKey, + url: `/${objectKey}?uploadId=${testUploadId}`, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + query: { uploadId: 'non-existent-upload-id' }, + actionImplicitDenies: false, + }; + assert.strictEqual(metadata.keyMaps.get(mpuBucket).size, 2); + multipartDelete(authInfo, deleteRequest, log, err => { + assert.strictEqual(err, null, `Expected no err but got ${err}`); + done(); + }); + }); }, - partHash, - }, postBody); - const partRequest2 = new DummyRequest({ - bucketName, - namespace, - objectKey, - headers: { - 'host': `${bucketName}.s3.amazonaws.com`, - 'content-length': '100', + ); + }, + ); + + it('should not leave orphans in data when overwriting an object with a MPU', done => { + const fullSizedPart = crypto.randomBytes(5 * 1024 * 1024); + const partBody = Buffer.from('I am a part\n', 'utf8'); + async.waterfall( + [ + next => bucketPut(authInfo, bucketPutRequest, log, next), + (corsHeaders, next) => initiateMultipartUpload(authInfo, initiateRequest, log, next), + (result, corsHeaders, next) => parseString(result, next), + (json, next) => { + const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; + const partRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, + query: { + partNumber: '1', + uploadId: testUploadId, + }, + }, + fullSizedPart, + ); + objectPutPart(authInfo, partRequest, undefined, log, (err, partpartHash) => { + assert.deepStrictEqual(err, null); + next(null, testUploadId, partpartHash); + }); }, - parsedContentLength: 100, - url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, - query: { - partNumber: '2', - uploadId: testUploadId, + (testUploadId, part1partHash, next) => { + const part2Request = new DummyRequest( + { + bucketName, + namespace, + objectKey, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, + query: { + partNumber: '2', + uploadId: testUploadId, + }, + }, + partBody, + ); + objectPutPart(authInfo, part2Request, undefined, log, (err, part2partHash) => { + assert.deepStrictEqual(err, null); + next(null, testUploadId, part1partHash, part2partHash); + }); }, - post: postBody, - partHash, - }, postBody); - objectPutPart(authInfo, partRequest1, undefined, log, () => { - objectPutPart(authInfo, partRequest2, undefined, log, () => { - const completeBody = '' + + (testUploadId, part1partHash, part2partHash, next) => { + const completeBody = + '' + '' + '1' + - `"${partHash}"` + + `"${part1partHash}"` + '' + '' + '2' + - `"${partHash}"` + + `"${part2partHash}"` + '' + ''; const completeRequest = { bucketName, namespace, objectKey, - headers: { host: `${bucketName}.s3.amazonaws.com` }, + parsedHost: 's3.amazonaws.com', url: `/${objectKey}?uploadId=${testUploadId}`, + headers: { host: `${bucketName}.s3.amazonaws.com` }, query: { uploadId: testUploadId }, post: completeBody, - partHash, actionImplicitDenies: false, }; - completeMultipartUpload(authInfo, - completeRequest, log, (err, result) => { - assert.strictEqual(err, null); - parseString(result, err => { - assert.strictEqual(err, null); - const MD = metadata.keyMaps - .get(bucketName) - .get(objectKey); - assert(MD); - assert.strictEqual(MD['content-length'], - 6000100); - done(); - }); - }); - }); - }); - }); - }); - - it('should set a canned ACL for a multipart upload', done => { - const initiateRequest = { - bucketName, - namespace, - objectKey, - headers: { - 'host': `${bucketName}.s3.amazonaws.com`, - 'x-amz-meta-stuff': 'I am some user metadata', - 'x-amz-acl': 'authenticated-read', - }, - url: `/${objectKey}?uploads`, - actionImplicitDenies: false, - }; - - async.waterfall([ - next => bucketPut(authInfo, bucketPutRequest, log, next), - (corsHeaders, next) => initiateMultipartUpload(authInfo, - initiateRequest, log, next), - (result, corsHeaders, next) => parseString(result, next), - ], - (err, json) => { - // Need to build request in here since do not have uploadId - // until here - const testUploadId = - json.InitiateMultipartUploadResult.UploadId[0]; - const md5Hash = crypto.createHash('md5'); - const bufferBody = Buffer.from(postBody); - md5Hash.update(bufferBody); - const partHash = md5Hash.digest('hex'); - const partRequest1 = new DummyRequest({ - bucketName, - namespace, - objectKey, - headers: { - 'host': `${bucketName}.s3.amazonaws.com`, - 'content-length': 6000000, - }, - url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, - query: { - partNumber: '1', - uploadId: testUploadId, + completeMultipartUpload(authInfo, completeRequest, log, (err, result) => { + assert.deepStrictEqual(err, null); + next(null, result); + }); }, - partHash, - }, postBody); - const partRequest2 = new DummyRequest({ - bucketName, - namespace, - objectKey, - headers: { - 'host': `${bucketName}.s3.amazonaws.com`, - 'content-length': 100, + (result, next) => { + assert.strictEqual(ds[0], undefined); + assert.deepStrictEqual(ds[1].value, fullSizedPart); + assert.deepStrictEqual(ds[2].value, partBody); + initiateMultipartUpload(authInfo, initiateRequest, log, next); }, - url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, - query: { - partNumber: '2', - uploadId: testUploadId, + (result, corsHeaders, next) => parseString(result, next), + (json, next) => { + const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; + const overwritePartBody = Buffer.from('I am an overwrite part\n', 'utf8'); + const md5Hash = crypto.createHash('md5').update(overwritePartBody); + const partHash = md5Hash.digest('hex'); + const partRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, + query: { + partNumber: '1', + uploadId: testUploadId, + }, + partHash, + }, + overwritePartBody, + ); + objectPutPart(authInfo, partRequest, undefined, log, () => next(null, testUploadId, partHash)); }, - partHash, - }, postBody); - objectPutPart(authInfo, partRequest1, undefined, log, () => { - objectPutPart(authInfo, partRequest2, undefined, log, () => { - const completeBody = '' + + (testUploadId, partHash, next) => { + const completeBody = + '' + '' + '1' + `"${partHash}"` + '' + - '' + - '2' + - `"${partHash}"` + - '' + ''; const completeRequest = { bucketName, namespace, objectKey, - headers: { host: `${bucketName}.s3.amazonaws.com` }, + parsedHost: 's3.amazonaws.com', url: `/${objectKey}?uploadId=${testUploadId}`, + headers: { host: `${bucketName}.s3.amazonaws.com` }, query: { uploadId: testUploadId }, post: completeBody, - partHash, actionImplicitDenies: false, }; - completeMultipartUpload(authInfo, - completeRequest, log, (err, result) => { - assert.strictEqual(err, null); - parseString(result, err => { - assert.strictEqual(err, null); - const MD = metadata.keyMaps - .get(bucketName) - .get(objectKey); - assert(MD); - assert.strictEqual(MD.acl.Canned, - 'authenticated-read'); - done(); - }); - }); - }); - }); - }); + completeMultipartUpload(authInfo, completeRequest, log, next); + }, + ], + err => { + assert.deepStrictEqual(err, null); + assert.strictEqual(ds[0], undefined); + assert.strictEqual(ds[1], undefined); + assert.strictEqual(ds[2], undefined); + assert.deepStrictEqual(ds[3].value, Buffer.from('I am an overwrite part\n', 'utf8')); + done(); + }, + ); }); - it('should set specific ACL grants for a multipart upload', done => { - const granteeId = '79a59df900b949e55d96a1e698fbace' + - 'dfd6e09d98eacf8f8d5218e7cd47ef2be'; - const granteeEmail = 'sampleAccount1@sampling.com'; - const initiateRequest = { - bucketName, - namespace, - objectKey, - headers: { - 'host': `${bucketName}.s3.amazonaws.com`, - 'x-amz-meta-stuff': 'I am some user metadata', - 'x-amz-grant-read': `emailAddress="${granteeEmail}"`, - }, - url: `/${objectKey}?uploads`, - actionImplicitDenies: false, - }; + it('should not leave orphans in data when overwriting an object part', done => { + const fullSizedPart = crypto.randomBytes(5 * 1024 * 1024); + const overWritePart = Buffer.from('Overwrite content', 'utf8'); + let uploadId; - async.waterfall([ - next => bucketPut(authInfo, bucketPutRequest, log, next), - (corsHeaders, next) => initiateMultipartUpload(authInfo, - initiateRequest, log, next), - (result, corsHeaders, next) => parseString(result, next), - ], - (err, json) => { - // Need to build request in here since do not have uploadId - // until here - const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; - const md5Hash = crypto.createHash('md5'); - const bufferBody = Buffer.from(postBody); - md5Hash.update(bufferBody); - const partHash = md5Hash.digest('hex'); - const partRequest1 = new DummyRequest({ - bucketName, - namespace, - objectKey, - headers: { - 'host': `${bucketName}.s3.amazonaws.com`, - 'content-length': 6000000, - }, - url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, - query: { - partNumber: '1', - uploadId: testUploadId, - }, - partHash, - }, postBody); - const partRequest2 = new DummyRequest({ - bucketName, - namespace, - objectKey, - headers: { - 'host': `${bucketName}.s3.amazonaws.com`, - 'content-length': 100, + async.waterfall( + [ + next => bucketPut(authInfo, bucketPutRequest, log, next), + (corsHeaders, next) => initiateMultipartUpload(authInfo, initiateRequest, log, next), + (result, corsHeaders, next) => parseString(result, next), + (json, next) => { + uploadId = json.InitiateMultipartUploadResult.UploadId[0]; + const requestObj = { + bucketName, + namespace, + objectKey, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: `/${objectKey}?partNumber=1&uploadId=${uploadId}`, + query: { + partNumber: '1', + uploadId, + }, + }; + const partRequest = new DummyRequest(requestObj, fullSizedPart); + objectPutPart(authInfo, partRequest, undefined, log, err => { + assert.deepStrictEqual(err, null); + next(null, requestObj); + }); }, - url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, - query: { - partNumber: '2', - uploadId: testUploadId, + (requestObj, next) => { + assert.deepStrictEqual(ds[1].value, fullSizedPart); + const partRequest = new DummyRequest(requestObj, overWritePart); + objectPutPart(authInfo, partRequest, undefined, log, (err, partpartHash) => { + assert.deepStrictEqual(err, null); + next(null, partpartHash); + }); }, - post: postBody, - partHash, - }, postBody); - objectPutPart(authInfo, partRequest1, undefined, log, () => { - objectPutPart(authInfo, partRequest2, undefined, log, () => { - const completeBody = '' + + (partpartHash, next) => { + const completeBody = + '' + '' + '1' + - `"${partHash}"` + - '' + - '' + - '2' + - `"${partHash}"` + + `"${partpartHash}"` + '' + ''; + const completeRequest = { bucketName, namespace, objectKey, + parsedHost: 's3.amazonaws.com', + url: `/${objectKey}?uploadId=${uploadId}`, headers: { host: `${bucketName}.s3.amazonaws.com` }, - url: `/${objectKey}?uploadId=${testUploadId}`, - query: { uploadId: testUploadId }, + query: { uploadId }, post: completeBody, - partHash, actionImplicitDenies: false, }; - completeMultipartUpload(authInfo, - completeRequest, log, (err, result) => { - assert.strictEqual(err, null); - parseString(result, err => { - assert.strictEqual(err, null); - const MD = metadata.keyMaps - .get(bucketName) - .get(objectKey); - assert(MD); - assert.strictEqual(MD.acl.READ[0], granteeId); - done(); - }); - }); - }); - }); - }); - }); - - it('should abort/delete a multipart upload', done => { - async.waterfall([ - next => bucketPut(authInfo, bucketPutRequest, log, next), - (corsHeaders, next) => initiateMultipartUpload(authInfo, - initiateRequest, log, next), - (result, corsHeaders, next) => parseString(result, next), - ], - (err, json) => { - // Need to build request in here since do not have uploadId - // until here - const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; - const bufferMD5 = Buffer.from(postBody, 'base64'); - const partHash = bufferMD5.toString('hex'); - const partRequest = new DummyRequest({ - bucketName, - namespace, - objectKey, - url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - query: { - partNumber: '1', - uploadId: testUploadId, - }, - partHash, - }, postBody); - objectPutPart(authInfo, partRequest, undefined, log, () => { - const deleteRequest = { - bucketName, - namespace, - objectKey, - url: `/${objectKey}?uploadId=${testUploadId}`, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - query: { uploadId: testUploadId }, - actionImplicitDenies: false, - }; - assert.strictEqual(metadata.keyMaps.get(mpuBucket).size, 2); - multipartDelete(authInfo, deleteRequest, log, err => { - assert.strictEqual(err, null); - assert.strictEqual(metadata.keyMaps.get(mpuBucket).size, 0); - done(); - }); - }); - }); - }); - - it('should return no error if attempt to abort/delete ' + - 'a multipart upload that does not exist and not using ' + - 'legacyAWSBehavior', done => { - async.waterfall([ - next => bucketPut(authInfo, bucketPutRequest, log, next), - (corsHeaders, next) => initiateMultipartUpload(authInfo, - initiateRequest, log, next), - (result, corsHeaders, next) => { - const mpuKeys = metadata.keyMaps.get(mpuBucket); - assert.strictEqual(mpuKeys.size, 1); - parseString(result, next); - }, - ], - (err, json) => { - // Need to build request in here since do not have uploadId - // until here - const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; - const bufferMD5 = Buffer.from(postBody, 'base64'); - const partHash = bufferMD5.toString('hex'); - const partRequest = new DummyRequest({ - bucketName, - namespace, - objectKey, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, - query: { - partNumber: '1', - uploadId: testUploadId, + completeMultipartUpload(authInfo, completeRequest, log, next); }, - partHash, - }, postBody); - objectPutPart(authInfo, partRequest, undefined, log, () => { - const deleteRequest = { - bucketName, - namespace, - objectKey, - url: `/${objectKey}?uploadId=${testUploadId}`, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - query: { uploadId: 'non-existent-upload-id' }, - actionImplicitDenies: false, - }; - assert.strictEqual(metadata.keyMaps.get(mpuBucket).size, 2); - multipartDelete(authInfo, deleteRequest, log, err => { - assert.strictEqual(err, null, - `Expected no err but got ${err}`); - done(); - }); - }); - }); - }); - - it('should not leave orphans in data when overwriting an object with a MPU', - done => { - const fullSizedPart = crypto.randomBytes(5 * 1024 * 1024); - const partBody = Buffer.from('I am a part\n', 'utf8'); - async.waterfall([ - next => bucketPut(authInfo, bucketPutRequest, log, next), - (corsHeaders, next) => initiateMultipartUpload(authInfo, - initiateRequest, log, next), - (result, corsHeaders, next) => parseString(result, next), - (json, next) => { - const testUploadId = - json.InitiateMultipartUploadResult.UploadId[0]; - const partRequest = new DummyRequest({ - bucketName, - namespace, - objectKey, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, - query: { - partNumber: '1', - uploadId: testUploadId, - }, - }, fullSizedPart); - objectPutPart(authInfo, partRequest, undefined, log, (err, - partpartHash) => { - assert.deepStrictEqual(err, null); - next(null, testUploadId, partpartHash); - }); - }, - (testUploadId, part1partHash, next) => { - const part2Request = new DummyRequest({ - bucketName, - namespace, - objectKey, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, - query: { - partNumber: '2', - uploadId: testUploadId, - }, - }, partBody); - objectPutPart(authInfo, part2Request, undefined, log, (err, - part2partHash) => { - assert.deepStrictEqual(err, null); - next(null, testUploadId, part1partHash, - part2partHash); - }); - }, - (testUploadId, part1partHash, part2partHash, next) => { - const completeBody = '' + - '' + - '1' + - `"${part1partHash}"` + - '' + - '' + - '2' + - `"${part2partHash}"` + - '' + - ''; - const completeRequest = { - bucketName, - namespace, - objectKey, - parsedHost: 's3.amazonaws.com', - url: `/${objectKey}?uploadId=${testUploadId}`, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - query: { uploadId: testUploadId }, - post: completeBody, - actionImplicitDenies: false, - }; - completeMultipartUpload(authInfo, completeRequest, log, - (err, result) => { - assert.deepStrictEqual(err, null); - next(null, result); - }); - }, - (result, next) => { + ], + err => { + assert.deepStrictEqual(err, null); assert.strictEqual(ds[0], undefined); - assert.deepStrictEqual(ds[1].value, fullSizedPart); - assert.deepStrictEqual(ds[2].value, partBody); - initiateMultipartUpload(authInfo, initiateRequest, log, next); - }, - (result, corsHeaders, next) => parseString(result, next), - (json, next) => { - const testUploadId = - json.InitiateMultipartUploadResult.UploadId[0]; - const overwritePartBody = - Buffer.from('I am an overwrite part\n', 'utf8'); - const md5Hash = crypto.createHash('md5') - .update(overwritePartBody); - const partHash = md5Hash.digest('hex'); - const partRequest = new DummyRequest({ - bucketName, - namespace, - objectKey, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, - query: { - partNumber: '1', - uploadId: testUploadId, - }, - partHash, - }, overwritePartBody); - objectPutPart(authInfo, partRequest, undefined, log, () => - next(null, testUploadId, partHash)); - }, - (testUploadId, partHash, next) => { - const completeBody = '' + - '' + - '1' + - `"${partHash}"` + - '' + - ''; - const completeRequest = { - bucketName, - namespace, - objectKey, - parsedHost: 's3.amazonaws.com', - url: `/${objectKey}?uploadId=${testUploadId}`, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - query: { uploadId: testUploadId }, - post: completeBody, - actionImplicitDenies: false, - }; - completeMultipartUpload(authInfo, completeRequest, log, next); + assert.deepStrictEqual(ds[1], undefined); + assert.deepStrictEqual(ds[2].value, overWritePart); + done(); }, - ], - err => { - assert.deepStrictEqual(err, null); - assert.strictEqual(ds[0], undefined); - assert.strictEqual(ds[1], undefined); - assert.strictEqual(ds[2], undefined); - assert.deepStrictEqual(ds[3].value, - Buffer.from('I am an overwrite part\n', 'utf8')); - done(); - }); + ); }); - it('should not leave orphans in data when overwriting an object part', - done => { + it('should leave orphaned data when overwriting an object part during completeMPU', done => { const fullSizedPart = crypto.randomBytes(5 * 1024 * 1024); const overWritePart = Buffer.from('Overwrite content', 'utf8'); let uploadId; - async.waterfall([ - next => bucketPut(authInfo, bucketPutRequest, log, next), - (corsHeaders, next) => initiateMultipartUpload(authInfo, - initiateRequest, log, next), - (result, corsHeaders, next) => parseString(result, next), - (json, next) => { - uploadId = json.InitiateMultipartUploadResult.UploadId[0]; - const requestObj = { - bucketName, - namespace, - objectKey, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - url: `/${objectKey}?partNumber=1&uploadId=${uploadId}`, - query: { - partNumber: '1', - uploadId, - }, - }; - const partRequest = new DummyRequest(requestObj, fullSizedPart); - objectPutPart(authInfo, partRequest, undefined, log, err => { - assert.deepStrictEqual(err, null); - next(null, requestObj); - }); - }, - (requestObj, next) => { - assert.deepStrictEqual(ds[1].value, fullSizedPart); - const partRequest = new DummyRequest(requestObj, overWritePart); - objectPutPart(authInfo, partRequest, undefined, log, - (err, partpartHash) => { + async.waterfall( + [ + next => bucketPut(authInfo, bucketPutRequest, log, next), + (corsHeaders, next) => initiateMultipartUpload(authInfo, initiateRequest, log, next), + (result, corsHeaders, next) => parseString(result, next), + (json, next) => { + uploadId = json.InitiateMultipartUploadResult.UploadId[0]; + const requestObj = { + bucketName, + namespace, + objectKey, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: `/${objectKey}?partNumber=1&uploadId=${uploadId}`, + query: { + partNumber: '1', + uploadId, + }, + }; + const partRequest = new DummyRequest(requestObj, fullSizedPart); + objectPutPart(authInfo, partRequest, undefined, log, (err, partpartHash) => { assert.deepStrictEqual(err, null); - next(null, partpartHash); + next(null, requestObj, partpartHash); }); - }, - (partpartHash, next) => { - const completeBody = '' + - '' + - '1' + - `"${partpartHash}"` + - '' + - ''; - - const completeRequest = { - bucketName, - namespace, - objectKey, - parsedHost: 's3.amazonaws.com', - url: `/${objectKey}?uploadId=${uploadId}`, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - query: { uploadId }, - post: completeBody, - actionImplicitDenies: false, - }; - completeMultipartUpload(authInfo, completeRequest, log, next); - }, - ], - err => { - assert.deepStrictEqual(err, null); - assert.strictEqual(ds[0], undefined); - assert.deepStrictEqual(ds[1], undefined); - assert.deepStrictEqual(ds[2].value, overWritePart); - done(); - }); - }); - - it('should leave orphaned data when overwriting an object part during completeMPU', - done => { - const fullSizedPart = crypto.randomBytes(5 * 1024 * 1024); - const overWritePart = Buffer.from('Overwrite content', 'utf8'); - let uploadId; - - async.waterfall([ - next => bucketPut(authInfo, bucketPutRequest, log, next), - (corsHeaders, next) => initiateMultipartUpload(authInfo, - initiateRequest, log, next), - (result, corsHeaders, next) => parseString(result, next), - (json, next) => { - uploadId = json.InitiateMultipartUploadResult.UploadId[0]; - const requestObj = { - bucketName, - namespace, - objectKey, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - url: `/${objectKey}?partNumber=1&uploadId=${uploadId}`, - query: { - partNumber: '1', - uploadId, - }, - }; - const partRequest = new DummyRequest(requestObj, fullSizedPart); - objectPutPart(authInfo, partRequest, undefined, log, (err, partpartHash) => { - assert.deepStrictEqual(err, null); - next(null, requestObj, partpartHash); - }); - }, - (requestObj, partpartHash, next) => { + }, + (requestObj, partpartHash, next) => { + assert.deepStrictEqual(ds[1].value, fullSizedPart); + async.parallel( + [ + done => { + const partRequest = new DummyRequest(requestObj, overWritePart); + objectPutPart(authInfo, partRequest, undefined, log, err => { + assert.deepStrictEqual(err, null); + done(); + }); + }, + done => { + const completeBody = + '' + + '' + + '1' + + `"${partpartHash}"` + + '' + + ''; + + const completeRequest = { + bucketName, + namespace, + objectKey, + parsedHost: 's3.amazonaws.com', + url: `/${objectKey}?uploadId=${uploadId}`, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + query: { uploadId }, + post: completeBody, + actionImplicitDenies: false, + }; + completeMultipartUpload(authInfo, completeRequest, log, done); + }, + ], + err => next(err), + ); + }, + ], + err => { + assert.deepStrictEqual(err, null); + assert.strictEqual(ds[0], undefined); assert.deepStrictEqual(ds[1].value, fullSizedPart); - async.parallel([ - done => { - const partRequest = new DummyRequest(requestObj, overWritePart); - objectPutPart(authInfo, partRequest, undefined, log, err => { - assert.deepStrictEqual(err, null); - done(); - }); - }, - done => { - const completeBody = '' + - '' + - '1' + - `"${partpartHash}"` + - '' + - ''; - - const completeRequest = { - bucketName, - namespace, - objectKey, - parsedHost: 's3.amazonaws.com', - url: `/${objectKey}?uploadId=${uploadId}`, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - query: { uploadId }, - post: completeBody, - actionImplicitDenies: false, - }; - completeMultipartUpload(authInfo, completeRequest, log, done); - }, - ], err => next(err)); - }, - ], - err => { - assert.deepStrictEqual(err, null); - assert.strictEqual(ds[0], undefined); - assert.deepStrictEqual(ds[1].value, fullSizedPart); - assert.deepStrictEqual(ds[2].value, overWritePart); - done(); - }); - }); - - it('should throw an error on put of an object part with an invalid ' + - 'uploadId', done => { - const testUploadId = 'invalidUploadID'; - const partRequest = new DummyRequest({ - bucketName, - url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, - query: { - partNumber: '1', - uploadId: testUploadId, + assert.deepStrictEqual(ds[2].value, overWritePart); + done(); }, - }, postBody); - - bucketPut(authInfo, bucketPutRequest, log, () => - objectPutPart(authInfo, partRequest, undefined, log, err => { - assert(err.is.NoSuchUpload); - done(); - }) ); }); - it('should complete an MPU with fewer parts than were originally ' + - 'put and delete data from left out parts', done => { - async.waterfall([ - next => bucketPut(authInfo, bucketPutRequest, log, next), - (corsHeaders, next) => initiateMultipartUpload(authInfo, - initiateRequest, log, next), - (result, corsHeaders, next) => parseString(result, next), - ], - (err, json) => { - // Need to build request in here since do not have uploadId - // until here - assert.ifError(err); - const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; - const fullSizedPart = crypto.randomBytes(5 * 1024 * 1024); - const partRequest1 = new DummyRequest({ + it('should throw an error on put of an object part with an invalid ' + 'uploadId', done => { + const testUploadId = 'invalidUploadID'; + const partRequest = new DummyRequest( + { bucketName, - namespace, - objectKey, url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, - headers: { host: `${bucketName}.s3.amazonaws.com` }, query: { partNumber: '1', uploadId: testUploadId, }, - }, fullSizedPart); - const partRequest2 = new DummyRequest({ - bucketName, - namespace, - objectKey, - url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - query: { - partNumber: '2', - uploadId: testUploadId, - }, - }, postBody); - objectPutPart(authInfo, partRequest1, undefined, log, err => { - assert.deepStrictEqual(err, null); - const md5Hash = crypto.createHash('md5').update(fullSizedPart); - const partHash = md5Hash.digest('hex'); - objectPutPart(authInfo, partRequest2, undefined, log, err => { - assert.deepStrictEqual(err, null); - const completeBody = '' + - '' + - '1' + - `"${partHash}"` + - '' + - ''; - const completeRequest = { - bucketName, - namespace, - objectKey, - url: `/${objectKey}?uploadId=${testUploadId}`, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - query: { uploadId: testUploadId }, - post: completeBody, - partHash, - actionImplicitDenies: false, - }; - // show that second part data is there - assert(ds[2]); - completeMultipartUpload(authInfo, - completeRequest, log, err => { - assert.strictEqual(err, null); - process.nextTick(() => { - // data has been deleted - assert.strictEqual(ds[2], undefined); - done(); + }, + postBody, + ); + + bucketPut(authInfo, bucketPutRequest, log, () => + objectPutPart(authInfo, partRequest, undefined, log, err => { + assert(err.is.NoSuchUpload); + done(); + }), + ); + }); + + it( + 'should complete an MPU with fewer parts than were originally ' + 'put and delete data from left out parts', + done => { + async.waterfall( + [ + next => bucketPut(authInfo, bucketPutRequest, log, next), + (corsHeaders, next) => initiateMultipartUpload(authInfo, initiateRequest, log, next), + (result, corsHeaders, next) => parseString(result, next), + ], + (err, json) => { + // Need to build request in here since do not have uploadId + // until here + assert.ifError(err); + const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; + const fullSizedPart = crypto.randomBytes(5 * 1024 * 1024); + const partRequest1 = new DummyRequest( + { + bucketName, + namespace, + objectKey, + url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + query: { + partNumber: '1', + uploadId: testUploadId, + }, + }, + fullSizedPart, + ); + const partRequest2 = new DummyRequest( + { + bucketName, + namespace, + objectKey, + url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + query: { + partNumber: '2', + uploadId: testUploadId, + }, + }, + postBody, + ); + objectPutPart(authInfo, partRequest1, undefined, log, err => { + assert.deepStrictEqual(err, null); + const md5Hash = crypto.createHash('md5').update(fullSizedPart); + const partHash = md5Hash.digest('hex'); + objectPutPart(authInfo, partRequest2, undefined, log, err => { + assert.deepStrictEqual(err, null); + const completeBody = + '' + + '' + + '1' + + `"${partHash}"` + + '' + + ''; + const completeRequest = { + bucketName, + namespace, + objectKey, + url: `/${objectKey}?uploadId=${testUploadId}`, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + query: { uploadId: testUploadId }, + post: completeBody, + partHash, + actionImplicitDenies: false, + }; + // show that second part data is there + assert(ds[2]); + completeMultipartUpload(authInfo, completeRequest, log, err => { + assert.strictEqual(err, null); + process.nextTick(() => { + // data has been deleted + assert.strictEqual(ds[2], undefined); + done(); + }); }); }); - }); - }); - }); - }); + }); + }, + ); + }, + ); - it('should not delete data locations on completeMultipartUpload retry', - done => { + it('should not delete data locations on completeMultipartUpload retry', done => { const partBody = Buffer.from('foo', 'utf8'); let origDeleteObject; - async.waterfall([ - next => - bucketPut(authInfo, bucketPutRequest, log, err => next(err)), - next => - initiateMultipartUpload(authInfo, initiateRequest, log, next), - (result, corsHeaders, next) => parseString(result, next), - (json, next) => { - const testUploadId = - json.InitiateMultipartUploadResult.UploadId[0]; - const partRequest = _createPutPartRequest(testUploadId, 1, - partBody); - objectPutPart(authInfo, partRequest, undefined, log, - (err, eTag) => next(err, eTag, testUploadId)); - }, - (eTag, testUploadId, next) => { - origDeleteObject = metadataBackend.deleteObject; - metadataBackend.deleteObject = ( - bucketName, objName, params, log, cb) => { + async.waterfall( + [ + next => bucketPut(authInfo, bucketPutRequest, log, err => next(err)), + next => initiateMultipartUpload(authInfo, initiateRequest, log, next), + (result, corsHeaders, next) => parseString(result, next), + (json, next) => { + const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; + const partRequest = _createPutPartRequest(testUploadId, 1, partBody); + objectPutPart(authInfo, partRequest, undefined, log, (err, eTag) => next(err, eTag, testUploadId)); + }, + (eTag, testUploadId, next) => { + origDeleteObject = metadataBackend.deleteObject; + metadataBackend.deleteObject = (bucketName, objName, params, log, cb) => { // prevent deletions from MPU bucket only - if (bucketName === mpuBucket) { - return process.nextTick( - () => cb(errors.InternalError)); - } - return origDeleteObject( - bucketName, objName, params, log, cb); - }; - const parts = [{ partNumber: 1, eTag }]; - const completeRequest = _createCompleteMpuRequest( - testUploadId, parts); - completeMultipartUpload(authInfo, completeRequest, log, err => { - // expect a failure here because we could not - // remove the overview key - assert(err.is.InternalError); - next(null, eTag, testUploadId); - }); - }, - (eTag, testUploadId, next) => { - // allow MPU bucket metadata deletions to happen again - metadataBackend.deleteObject = origDeleteObject; - // retry the completeMultipartUpload with the same - // metadata, as an application would normally do after - // a failure - const parts = [{ partNumber: 1, eTag }]; - const completeRequest = _createCompleteMpuRequest( - testUploadId, parts); - completeMultipartUpload(authInfo, completeRequest, log, next); + if (bucketName === mpuBucket) { + return process.nextTick(() => cb(errors.InternalError)); + } + return origDeleteObject(bucketName, objName, params, log, cb); + }; + const parts = [{ partNumber: 1, eTag }]; + const completeRequest = _createCompleteMpuRequest(testUploadId, parts); + completeMultipartUpload(authInfo, completeRequest, log, err => { + // expect a failure here because we could not + // remove the overview key + assert(err.is.InternalError); + next(null, eTag, testUploadId); + }); + }, + (eTag, testUploadId, next) => { + // allow MPU bucket metadata deletions to happen again + metadataBackend.deleteObject = origDeleteObject; + // retry the completeMultipartUpload with the same + // metadata, as an application would normally do after + // a failure + const parts = [{ partNumber: 1, eTag }]; + const completeRequest = _createCompleteMpuRequest(testUploadId, parts); + completeMultipartUpload(authInfo, completeRequest, log, next); + }, + ], + err => { + assert.ifError(err); + // check that the original data has not been deleted + // during the replay + assert.strictEqual(ds[0], undefined); + assert.notStrictEqual(ds[1], undefined); + assert.deepStrictEqual(ds[1].value, partBody); + done(); }, - ], err => { - assert.ifError(err); - // check that the original data has not been deleted - // during the replay - assert.strictEqual(ds[0], undefined); - assert.notStrictEqual(ds[1], undefined); - assert.deepStrictEqual(ds[1].value, partBody); - done(); - }); + ); }); it('should abort an MPU and delete its MD if it has been created by a failed complete before', done => { const delMeta = metadataBackend.deleteObject; metadataBackend.deleteObject = (bucketName, objName, params, log, cb) => cb(errors.InternalError); const partBody = Buffer.from('I am a part\n', 'utf8'); - initiateRequest.headers['x-amz-meta-stuff'] = - 'I am some user metadata'; - async.waterfall([ - next => bucketPut(authInfo, bucketPutRequest, log, next), - (corsHeaders, next) => initiateMultipartUpload(authInfo, - initiateRequest, log, next), - (result, corsHeaders, next) => parseString(result, next), - ], - (err, json) => { - assert.ifError(err); - const testUploadId = - json.InitiateMultipartUploadResult.UploadId[0]; - const partHash = crypto.createHash('md5').update(partBody).digest('hex'); - const partRequest = new DummyRequest({ - bucketName, - namespace, - objectKey, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, - query: { - partNumber: '1', - uploadId: testUploadId, - }, - partHash, - }, partBody); - objectPutPart(authInfo, partRequest, undefined, log, () => { - const completeBody = '' + - '' + - '1' + - `"${partHash}"` + - '' + - ''; - const completeRequest = { - bucketName, - namespace, - objectKey, - parsedHost: 's3.amazonaws.com', - url: `/${objectKey}?uploadId=${testUploadId}`, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - query: { uploadId: testUploadId }, - post: completeBody, - actionImplicitDenies: false, - }; - completeMultipartUpload(authInfo, - completeRequest, log, err => { + initiateRequest.headers['x-amz-meta-stuff'] = 'I am some user metadata'; + async.waterfall( + [ + next => bucketPut(authInfo, bucketPutRequest, log, next), + (corsHeaders, next) => initiateMultipartUpload(authInfo, initiateRequest, log, next), + (result, corsHeaders, next) => parseString(result, next), + ], + (err, json) => { + assert.ifError(err); + const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; + const partHash = crypto.createHash('md5').update(partBody).digest('hex'); + const partRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, + query: { + partNumber: '1', + uploadId: testUploadId, + }, + partHash, + }, + partBody, + ); + objectPutPart(authInfo, partRequest, undefined, log, () => { + const completeBody = + '' + + '' + + '1' + + `"${partHash}"` + + '' + + ''; + const completeRequest = { + bucketName, + namespace, + objectKey, + parsedHost: 's3.amazonaws.com', + url: `/${objectKey}?uploadId=${testUploadId}`, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + query: { uploadId: testUploadId }, + post: completeBody, + actionImplicitDenies: false, + }; + completeMultipartUpload(authInfo, completeRequest, log, err => { assert(err.is.InternalError); - const MD = metadata.keyMaps.get(bucketName) - .get(objectKey); + const MD = metadata.keyMaps.get(bucketName).get(objectKey); assert(MD); - assert.strictEqual(MD['x-amz-meta-stuff'], - 'I am some user metadata'); + assert.strictEqual(MD['x-amz-meta-stuff'], 'I am some user metadata'); assert.strictEqual(MD.uploadId, testUploadId); metadataBackend.deleteObject = delMeta; @@ -2035,145 +2099,155 @@ describe('Multipart Upload API', () => { done(); }); }); - }); - }); + }); + }, + ); }); it('should complete an MPU and promote its MD if it has been created by a failed complete before', done => { const delMeta = metadataBackend.deleteObject; metadataBackend.deleteObject = (bucketName, objName, params, log, cb) => cb(errors.InternalError); const partBody = Buffer.from('I am a part\n', 'utf8'); - initiateRequest.headers['x-amz-meta-stuff'] = - 'I am some user metadata'; - async.waterfall([ - next => bucketPut(authInfo, bucketPutRequest, log, next), - (corsHeaders, next) => initiateMultipartUpload(authInfo, - initiateRequest, log, next), - (result, corsHeaders, next) => parseString(result, next), - ], - (err, json) => { - assert.ifError(err); - const testUploadId = - json.InitiateMultipartUploadResult.UploadId[0]; - const partHash = crypto.createHash('md5').update(partBody).digest('hex'); - const partRequest = new DummyRequest({ - bucketName, - namespace, - objectKey, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, - query: { - partNumber: '1', - uploadId: testUploadId, - }, - partHash, - }, partBody); - objectPutPart(authInfo, partRequest, undefined, log, () => { - const completeBody = '' + - '' + - '1' + - `"${partHash}"` + - '' + - ''; - const completeRequest = { - bucketName, - namespace, - objectKey, - parsedHost: 's3.amazonaws.com', - url: `/${objectKey}?uploadId=${testUploadId}`, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - query: { uploadId: testUploadId }, - post: completeBody, - actionImplicitDenies: false, - }; - completeMultipartUpload(authInfo, - completeRequest, log, err => { + initiateRequest.headers['x-amz-meta-stuff'] = 'I am some user metadata'; + async.waterfall( + [ + next => bucketPut(authInfo, bucketPutRequest, log, next), + (corsHeaders, next) => initiateMultipartUpload(authInfo, initiateRequest, log, next), + (result, corsHeaders, next) => parseString(result, next), + ], + (err, json) => { + assert.ifError(err); + const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; + const partHash = crypto.createHash('md5').update(partBody).digest('hex'); + const partRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, + query: { + partNumber: '1', + uploadId: testUploadId, + }, + partHash, + }, + partBody, + ); + objectPutPart(authInfo, partRequest, undefined, log, () => { + const completeBody = + '' + + '' + + '1' + + `"${partHash}"` + + '' + + ''; + const completeRequest = { + bucketName, + namespace, + objectKey, + parsedHost: 's3.amazonaws.com', + url: `/${objectKey}?uploadId=${testUploadId}`, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + query: { uploadId: testUploadId }, + post: completeBody, + actionImplicitDenies: false, + }; + completeMultipartUpload(authInfo, completeRequest, log, err => { assert(err.is.InternalError); const MD = metadata.keyMaps.get(bucketName).get(objectKey); assert(MD); - assert.strictEqual(MD['x-amz-meta-stuff'], - 'I am some user metadata'); + assert.strictEqual(MD['x-amz-meta-stuff'], 'I am some user metadata'); assert.strictEqual(MD.uploadId, testUploadId); metadataBackend.deleteObject = delMeta; assert.strictEqual(metadata.keyMaps.get(mpuBucket).size, 2); - completeMultipartUpload(authInfo, - completeRequest, log, err => { - assert.ifError(err); - const MD = metadata.keyMaps.get(bucketName) - .get(objectKey); - assert(MD); - assert.strictEqual(MD['x-amz-meta-stuff'], - 'I am some user metadata'); - assert.strictEqual(metadata.keyMaps.get(mpuBucket).size, 0); - done(); - }); + completeMultipartUpload(authInfo, completeRequest, log, err => { + assert.ifError(err); + const MD = metadata.keyMaps.get(bucketName).get(objectKey); + assert(MD); + assert.strictEqual(MD['x-amz-meta-stuff'], 'I am some user metadata'); + assert.strictEqual(metadata.keyMaps.get(mpuBucket).size, 0); + done(); + }); }); - }); - }); + }); + }, + ); }); it('should not pass needOplogUpdate when writing new object', done => { - async.series([ - next => bucketPut(authInfo, bucketPutRequest, log, next), - async () => _uploadMpuObject(), - async () => { - const options = metadataswitch.putObjectMD.lastCall.args[3]; - assert.strictEqual(options.needOplogUpdate, undefined); - assert.strictEqual(options.originOp, undefined); - }, - ], done); + async.series( + [ + next => bucketPut(authInfo, bucketPutRequest, log, next), + async () => _uploadMpuObject(), + async () => { + const options = metadataswitch.putObjectMD.lastCall.args[3]; + assert.strictEqual(options.needOplogUpdate, undefined); + assert.strictEqual(options.originOp, undefined); + }, + ], + done, + ); }); it('should not pass needOplogUpdate when replacing object', done => { - async.series([ - next => bucketPut(authInfo, bucketPutRequest, log, next), - async () => _uploadMpuObject(), - async () => _uploadMpuObject(), - async () => { - const options = metadataswitch.putObjectMD.lastCall.args[3]; - assert.strictEqual(options.needOplogUpdate, undefined); - assert.strictEqual(options.originOp, undefined); - }, - ], done); + async.series( + [ + next => bucketPut(authInfo, bucketPutRequest, log, next), + async () => _uploadMpuObject(), + async () => _uploadMpuObject(), + async () => { + const options = metadataswitch.putObjectMD.lastCall.args[3]; + assert.strictEqual(options.needOplogUpdate, undefined); + assert.strictEqual(options.originOp, undefined); + }, + ], + done, + ); }); it('should pass needOplogUpdate to metadata when replacing archived object', done => { const archived = { - archiveInfo: { foo: 0, bar: 'stuff' } + archiveInfo: { foo: 0, bar: 'stuff' }, }; - async.series([ - next => bucketPut(authInfo, bucketPutRequest, log, next), - async () => _uploadMpuObject(), - next => fakeMetadataArchive(bucketName, objectKey, undefined, archived, next), - async () => _uploadMpuObject(), - async () => { - const options = metadataswitch.putObjectMD.lastCall.args[3]; - assert.strictEqual(options.needOplogUpdate, true); - assert.strictEqual(options.originOp, 's3:ReplaceArchivedObject'); - }, - ], done); + async.series( + [ + next => bucketPut(authInfo, bucketPutRequest, log, next), + async () => _uploadMpuObject(), + next => fakeMetadataArchive(bucketName, objectKey, undefined, archived, next), + async () => _uploadMpuObject(), + async () => { + const options = metadataswitch.putObjectMD.lastCall.args[3]; + assert.strictEqual(options.needOplogUpdate, true); + assert.strictEqual(options.originOp, 's3:ReplaceArchivedObject'); + }, + ], + done, + ); }); it('should pass needOplogUpdate to metadata when replacing archived object in version suspended bucket', done => { const archived = { - archiveInfo: { foo: 0, bar: 'stuff' } + archiveInfo: { foo: 0, bar: 'stuff' }, }; - const suspendVersioningRequest = versioningTestUtils - .createBucketPutVersioningReq(bucketName, 'Suspended'); - async.series([ - next => bucketPut(authInfo, bucketPutRequest, log, next), - next => bucketPutVersioning(authInfo, suspendVersioningRequest, log, next), - async () => _uploadMpuObject(), - next => fakeMetadataArchive(bucketName, objectKey, undefined, archived, next), - async () => _uploadMpuObject(), - async () => { - const options = metadataswitch.putObjectMD.lastCall.args[3]; - assert.strictEqual(options.needOplogUpdate, true); - assert.strictEqual(options.originOp, 's3:ReplaceArchivedObject'); - }, - ], done); + const suspendVersioningRequest = versioningTestUtils.createBucketPutVersioningReq(bucketName, 'Suspended'); + async.series( + [ + next => bucketPut(authInfo, bucketPutRequest, log, next), + next => bucketPutVersioning(authInfo, suspendVersioningRequest, log, next), + async () => _uploadMpuObject(), + next => fakeMetadataArchive(bucketName, objectKey, undefined, archived, next), + async () => _uploadMpuObject(), + async () => { + const options = metadataswitch.putObjectMD.lastCall.args[3]; + assert.strictEqual(options.needOplogUpdate, true); + assert.strictEqual(options.originOp, 's3:ReplaceArchivedObject'); + }, + ], + done, + ); }); it('should fail to initiate a multipart upload if location constraint is crr', done => { @@ -2194,11 +2268,10 @@ describe('Multipart Upload API', () => { bucketPut(authInfo, bucketPutRequest, log, err => { assert.ifError(err); - initiateMultipartUpload(authInfo, initiateRequest, - log, err => { - assert(err.is.InvalidArgument); - done(); - }); + initiateMultipartUpload(authInfo, initiateRequest, log, err => { + assert(err.is.InvalidArgument); + done(); + }); }); }); @@ -2206,72 +2279,78 @@ describe('Multipart Upload API', () => { const partBody = Buffer.from('I am a part\n', 'utf8'); let batchDeleteStub; - async.waterfall([ - next => bucketPut(authInfo, bucketPutRequest, log, next), - (corsHeaders, next) => initiateMultipartUpload(authInfo, - initiateRequest, log, next), - (result, corsHeaders, next) => parseString(result, next), - ], - (err, json) => { - assert.ifError(err); - const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; - const partHash = crypto.createHash('md5').update(partBody).digest('hex'); - const partRequest = new DummyRequest({ - bucketName, - namespace, - objectKey, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, - query: { - partNumber: '1', - uploadId: testUploadId, - }, - partHash, - }, partBody); - - objectPutPart(authInfo, partRequest, undefined, log, () => { - // Mock batchDeleteObjectMetadata to fail with non-retryable error - const services = require('../../../lib/services'); - batchDeleteStub = sinon.stub(services, 'batchDeleteObjectMetadata') - .callsFake((mpuBucketName, keysToDelete, log, cb) => - // Simulate a non-retryable error that should be converted to retryable - cb(errors.NoSuchKey) - ); + async.waterfall( + [ + next => bucketPut(authInfo, bucketPutRequest, log, next), + (corsHeaders, next) => initiateMultipartUpload(authInfo, initiateRequest, log, next), + (result, corsHeaders, next) => parseString(result, next), + ], + (err, json) => { + assert.ifError(err); + const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; + const partHash = crypto.createHash('md5').update(partBody).digest('hex'); + const partRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, + query: { + partNumber: '1', + uploadId: testUploadId, + }, + partHash, + }, + partBody, + ); + + objectPutPart(authInfo, partRequest, undefined, log, () => { + // Mock batchDeleteObjectMetadata to fail with non-retryable error + const services = require('../../../lib/services'); + batchDeleteStub = sinon + .stub(services, 'batchDeleteObjectMetadata') + .callsFake((mpuBucketName, keysToDelete, log, cb) => + // Simulate a non-retryable error that should be converted to retryable + cb(errors.NoSuchKey), + ); + + const completeBody = + '' + + '' + + '1' + + `"${partHash}"` + + '' + + ''; + const completeRequest = { + bucketName, + namespace, + objectKey, + parsedHost: 's3.amazonaws.com', + url: `/${objectKey}?uploadId=${testUploadId}`, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + query: { uploadId: testUploadId }, + post: completeBody, + actionImplicitDenies: false, + }; - const completeBody = '' + - '' + - '1' + - `"${partHash}"` + - '' + - ''; - const completeRequest = { - bucketName, - namespace, - objectKey, - parsedHost: 's3.amazonaws.com', - url: `/${objectKey}?uploadId=${testUploadId}`, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - query: { uploadId: testUploadId }, - post: completeBody, - actionImplicitDenies: false, - }; - - completeMultipartUpload(authInfo, completeRequest, log, err => { - // Restore original function - batchDeleteStub.restore(); - - // Should get an error (retryable behavior) - assert(err, 'Expected an error when metadata deletion fails'); - - // Verify S3 object was created successfully despite the error - const objMD = metadata.keyMaps.get(bucketName).get(objectKey); - assert(objMD, 'S3 object should exist even when metadata cleanup fails'); - assert.strictEqual(objMD.uploadId, testUploadId); + completeMultipartUpload(authInfo, completeRequest, log, err => { + // Restore original function + batchDeleteStub.restore(); - done(); + // Should get an error (retryable behavior) + assert(err, 'Expected an error when metadata deletion fails'); + + // Verify S3 object was created successfully despite the error + const objMD = metadata.keyMaps.get(bucketName).get(objectKey); + assert(objMD, 'S3 object should exist even when metadata cleanup fails'); + assert.strictEqual(objMD.uploadId, testUploadId); + + done(); + }); }); - }); - }); + }, + ); }); it('should not return error if batchDeleteExtraParts fails', done => { @@ -2279,115 +2358,123 @@ describe('Multipart Upload API', () => { const partBody = Buffer.from('I am a smaller part\n', 'utf8'); let batchDeleteStub; - async.waterfall([ - next => bucketPut(authInfo, bucketPutRequest, log, next), - (corsHeaders, next) => initiateMultipartUpload(authInfo, - initiateRequest, log, next), - (result, corsHeaders, next) => parseString(result, next), - ], - (err, json) => { - assert.ifError(err); - const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; - - // Upload part 1 (will be included in completion) - const partRequest1 = new DummyRequest({ - bucketName, - namespace, - objectKey, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, - query: { - partNumber: '1', - uploadId: testUploadId, - }, - }, fullSizedPart); - - objectPutPart(authInfo, partRequest1, undefined, log, (err, part1ETag) => { + async.waterfall( + [ + next => bucketPut(authInfo, bucketPutRequest, log, next), + (corsHeaders, next) => initiateMultipartUpload(authInfo, initiateRequest, log, next), + (result, corsHeaders, next) => parseString(result, next), + ], + (err, json) => { assert.ifError(err); - - // Upload part 2 (will be an "extra part" not included in completion) - const partRequest2 = new DummyRequest({ - bucketName, - namespace, - objectKey, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - url: `/${objectKey}?partNumber=2&uploadId=${testUploadId}`, - query: { - partNumber: '2', - uploadId: testUploadId, - }, - }, partBody); - - objectPutPart(authInfo, partRequest2, undefined, log, err => { - assert.ifError(err); - - // Mock data.batchDelete to fail when deleting extra parts - const { data } = require('../../../lib/data/wrapper'); - batchDeleteStub = sinon.stub(data, 'batchDelete') - .callsFake((locations, method, dataStoreName, log, cb) => - // Always fail extra part deletion - cb(new Error('Simulated extra part deletion failure')) - ); - - // Complete MPU with only part 1 (part 2 becomes "extra part") - const completeBody = '' + - '' + - '1' + - `"${part1ETag}"` + - '' + - ''; - const completeRequest = { + const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; + + // Upload part 1 (will be included in completion) + const partRequest1 = new DummyRequest( + { bucketName, namespace, objectKey, - parsedHost: 's3.amazonaws.com', - url: `/${objectKey}?uploadId=${testUploadId}`, headers: { host: `${bucketName}.s3.amazonaws.com` }, - query: { uploadId: testUploadId }, - post: completeBody, - actionImplicitDenies: false, - }; + url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, + query: { + partNumber: '1', + uploadId: testUploadId, + }, + }, + fullSizedPart, + ); - completeMultipartUpload(authInfo, completeRequest, log, err => { - // Restore original function - batchDeleteStub.restore(); + objectPutPart(authInfo, partRequest1, undefined, log, (err, part1ETag) => { + assert.ifError(err); + + // Upload part 2 (will be an "extra part" not included in completion) + const partRequest2 = new DummyRequest( + { + bucketName, + namespace, + objectKey, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: `/${objectKey}?partNumber=2&uploadId=${testUploadId}`, + query: { + partNumber: '2', + uploadId: testUploadId, + }, + }, + partBody, + ); - // Should NOT get an error despite extra part deletion failing - assert.ifError(err, 'Should not return error when extra part deletion fails'); + objectPutPart(authInfo, partRequest2, undefined, log, err => { + assert.ifError(err); - // Verify S3 object was created successfully - const objMD = metadata.keyMaps.get(bucketName).get(objectKey); - assert(objMD, 'S3 object should exist'); - assert.strictEqual(objMD.uploadId, testUploadId); + // Mock data.batchDelete to fail when deleting extra parts + const { data } = require('../../../lib/data/wrapper'); + batchDeleteStub = sinon + .stub(data, 'batchDelete') + .callsFake((locations, method, dataStoreName, log, cb) => + // Always fail extra part deletion + cb(new Error('Simulated extra part deletion failure')), + ); + + // Complete MPU with only part 1 (part 2 becomes "extra part") + const completeBody = + '' + + '' + + '1' + + `"${part1ETag}"` + + '' + + ''; + const completeRequest = { + bucketName, + namespace, + objectKey, + parsedHost: 's3.amazonaws.com', + url: `/${objectKey}?uploadId=${testUploadId}`, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + query: { uploadId: testUploadId }, + post: completeBody, + actionImplicitDenies: false, + }; - // Verify MPU metadata was cleaned up - assert.strictEqual(metadata.keyMaps.get(mpuBucket).size, 0, - 'MPU metadata should be cleaned up'); + completeMultipartUpload(authInfo, completeRequest, log, err => { + // Restore original function + batchDeleteStub.restore(); - done(); + // Should NOT get an error despite extra part deletion failing + assert.ifError(err, 'Should not return error when extra part deletion fails'); + + // Verify S3 object was created successfully + const objMD = metadata.keyMaps.get(bucketName).get(objectKey); + assert(objMD, 'S3 object should exist'); + assert.strictEqual(objMD.uploadId, testUploadId); + + // Verify MPU metadata was cleaned up + assert.strictEqual( + metadata.keyMaps.get(mpuBucket).size, + 0, + 'MPU metadata should be cleaned up', + ); + + done(); + }); }); }); - }); - }); + }, + ); }); }); describe('complete mpu with versioning', () => { - const objData = ['foo0', 'foo1', 'foo2'].map(str => - Buffer.from(str, 'utf8')); + const objData = ['foo0', 'foo1', 'foo2'].map(str => Buffer.from(str, 'utf8')); - const enableVersioningRequest = - versioningTestUtils.createBucketPutVersioningReq(bucketName, 'Enabled'); - const suspendVersioningRequest = versioningTestUtils - .createBucketPutVersioningReq(bucketName, 'Suspended'); + const enableVersioningRequest = versioningTestUtils.createBucketPutVersioningReq(bucketName, 'Enabled'); + const suspendVersioningRequest = versioningTestUtils.createBucketPutVersioningReq(bucketName, 'Suspended'); let testPutObjectRequests; beforeEach(done => { cleanup(); testPutObjectRequests = objData - .slice(0, 2) - .map(data => versioningTestUtils.createPutObjectRequest( - bucketName, objectKey, data)); + .slice(0, 2) + .map(data => versioningTestUtils.createPutObjectRequest(bucketName, objectKey, data)); bucketPut(authInfo, bucketPutRequest, log, done); }); @@ -2396,291 +2483,271 @@ describe('complete mpu with versioning', () => { done(); }); - it('should delete null version when creating new null version, ' + - 'when null version is the latest version', done => { - async.waterfall([ - next => bucketPutVersioning(authInfo, - suspendVersioningRequest, log, err => next(err)), - next => initiateMultipartUpload( - authInfo, initiateRequest, log, next), - (result, corsHeaders, next) => parseString(result, next), - (json, next) => { - const partBody = objData[2]; - const testUploadId = - json.InitiateMultipartUploadResult.UploadId[0]; - const partRequest = _createPutPartRequest(testUploadId, 1, - partBody); - objectPutPart(authInfo, partRequest, undefined, log, - (err, eTag) => next(err, eTag, testUploadId)); - }, - (eTag, testUploadId, next) => { - const origPutObject = metadataBackend.putObject; - let callCount = 0; - metadataBackend.putObject = - (putBucketName, objName, objVal, params, log, cb) => { - if (callCount === 0) { - // first putObject sets the completeInProgress flag in the overview key - assert.strictEqual(putBucketName, `${constants.mpuBucketPrefix}${bucketName}`); - assert.strictEqual( - objName, `overview${splitter}${objectKey}${splitter}${testUploadId}`); - assert.strictEqual(objVal.completeInProgress, true); - } else { - assert.strictEqual(params.replayId, testUploadId); - assert.strictEqual(objVal.originOp, 's3:ObjectCreated:CompleteMultipartUpload'); + it( + 'should delete null version when creating new null version, ' + 'when null version is the latest version', + done => { + async.waterfall( + [ + next => bucketPutVersioning(authInfo, suspendVersioningRequest, log, err => next(err)), + next => initiateMultipartUpload(authInfo, initiateRequest, log, next), + (result, corsHeaders, next) => parseString(result, next), + (json, next) => { + const partBody = objData[2]; + const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; + const partRequest = _createPutPartRequest(testUploadId, 1, partBody); + objectPutPart(authInfo, partRequest, undefined, log, (err, eTag) => + next(err, eTag, testUploadId), + ); + }, + (eTag, testUploadId, next) => { + const origPutObject = metadataBackend.putObject; + let callCount = 0; + metadataBackend.putObject = (putBucketName, objName, objVal, params, log, cb) => { + if (callCount === 0) { + // first putObject sets the completeInProgress flag in the overview key + assert.strictEqual(putBucketName, `${constants.mpuBucketPrefix}${bucketName}`); + assert.strictEqual( + objName, + `overview${splitter}${objectKey}${splitter}${testUploadId}`, + ); + assert.strictEqual(objVal.completeInProgress, true); + } else { + assert.strictEqual(params.replayId, testUploadId); + assert.strictEqual(objVal.originOp, 's3:ObjectCreated:CompleteMultipartUpload'); + metadataBackend.putObject = origPutObject; + } + origPutObject(putBucketName, objName, objVal, params, log, cb); + callCount += 1; + }; + const parts = [{ partNumber: 1, eTag }]; + const completeRequest = _createCompleteMpuRequest(testUploadId, parts); + completeMultipartUpload(authInfo, completeRequest, log, err => next(err, testUploadId)); + }, + (testUploadId, next) => { + const origPutObject = metadataBackend.putObject; + metadataBackend.putObject = (putBucketName, objName, objVal, params, log, cb) => { + assert.strictEqual(params.oldReplayId, testUploadId); + assert.strictEqual(objVal.originOp, 's3:ObjectCreated:Put'); metadataBackend.putObject = origPutObject; - } - origPutObject( - putBucketName, objName, objVal, params, log, cb); - callCount += 1; - }; - const parts = [{ partNumber: 1, eTag }]; - const completeRequest = _createCompleteMpuRequest(testUploadId, - parts); - completeMultipartUpload(authInfo, completeRequest, log, - err => next(err, testUploadId)); - }, - (testUploadId, next) => { - const origPutObject = metadataBackend.putObject; - metadataBackend.putObject = - (putBucketName, objName, objVal, params, log, cb) => { - assert.strictEqual(params.oldReplayId, testUploadId); - assert.strictEqual(objVal.originOp, 's3:ObjectCreated:Put'); - metadataBackend.putObject = origPutObject; - origPutObject( - putBucketName, objName, objVal, params, log, cb); - }; - // overwrite null version with a non-MPU object - objectPut(authInfo, testPutObjectRequests[1], - undefined, log, err => next(err)); - }, - ], err => { - assert.ifError(err, `Unexpected err: ${err}`); - done(); - }); - }); + origPutObject(putBucketName, objName, objVal, params, log, cb); + }; + // overwrite null version with a non-MPU object + objectPut(authInfo, testPutObjectRequests[1], undefined, log, err => next(err)); + }, + ], + err => { + assert.ifError(err, `Unexpected err: ${err}`); + done(); + }, + ); + }, + ); + + it( + 'should delete null version when creating new null version, ' + 'when null version is not the latest version', + done => { + async.waterfall( + [ + // putting null version: put obj before versioning configured + next => objectPut(authInfo, testPutObjectRequests[0], undefined, log, err => next(err)), + next => bucketPutVersioning(authInfo, enableVersioningRequest, log, err => next(err)), + // put another version: + next => objectPut(authInfo, testPutObjectRequests[1], undefined, log, err => next(err)), + next => bucketPutVersioning(authInfo, suspendVersioningRequest, log, err => next(err)), + next => { + versioningTestUtils.assertDataStoreValues(ds, objData.slice(0, 2)); + initiateMultipartUpload(authInfo, initiateRequest, log, next); + }, + (result, corsHeaders, next) => parseString(result, next), + (json, next) => { + const partBody = objData[2]; + const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; + const partRequest = _createPutPartRequest(testUploadId, 1, partBody); + objectPutPart(authInfo, partRequest, undefined, log, (err, eTag) => + next(err, eTag, testUploadId), + ); + }, + (eTag, testUploadId, next) => { + const origPutObject = metadataBackend.putObject; + let callCount = 0; + metadataBackend.putObject = (putBucketName, objName, objVal, params, log, cb) => { + if (callCount === 0) { + // first putObject sets the completeInProgress flag in the overview key + assert.strictEqual(putBucketName, `${constants.mpuBucketPrefix}${bucketName}`); + assert.strictEqual( + objName, + `overview${splitter}${objectKey}${splitter}${testUploadId}`, + ); + assert.strictEqual(objVal.completeInProgress, true); + } else { + assert.strictEqual(params.replayId, testUploadId); + metadataBackend.putObject = origPutObject; + } + origPutObject(putBucketName, objName, objVal, params, log, cb); + callCount += 1; + }; + const parts = [{ partNumber: 1, eTag }]; + const completeRequest = _createCompleteMpuRequest(testUploadId, parts); + completeMultipartUpload(authInfo, completeRequest, log, err => next(err, testUploadId)); + }, + (testUploadId, next) => { + versioningTestUtils.assertDataStoreValues(ds, [undefined, objData[1], objData[2]]); - it('should delete null version when creating new null version, ' + - 'when null version is not the latest version', done => { - async.waterfall([ - // putting null version: put obj before versioning configured - next => objectPut(authInfo, testPutObjectRequests[0], - undefined, log, err => next(err)), - next => bucketPutVersioning(authInfo, - enableVersioningRequest, log, err => next(err)), - // put another version: - next => objectPut(authInfo, testPutObjectRequests[1], - undefined, log, err => next(err)), - next => bucketPutVersioning(authInfo, - suspendVersioningRequest, log, err => next(err)), - next => { - versioningTestUtils.assertDataStoreValues( - ds, objData.slice(0, 2)); - initiateMultipartUpload(authInfo, initiateRequest, log, next); - }, - (result, corsHeaders, next) => parseString(result, next), - (json, next) => { - const partBody = objData[2]; - const testUploadId = - json.InitiateMultipartUploadResult.UploadId[0]; - const partRequest = _createPutPartRequest(testUploadId, 1, - partBody); - objectPutPart(authInfo, partRequest, undefined, log, - (err, eTag) => next(err, eTag, testUploadId)); - }, - (eTag, testUploadId, next) => { - const origPutObject = metadataBackend.putObject; - let callCount = 0; - metadataBackend.putObject = - (putBucketName, objName, objVal, params, log, cb) => { - if (callCount === 0) { - // first putObject sets the completeInProgress flag in the overview key - assert.strictEqual(putBucketName, `${constants.mpuBucketPrefix}${bucketName}`); - assert.strictEqual( - objName, `overview${splitter}${objectKey}${splitter}${testUploadId}`); - assert.strictEqual(objVal.completeInProgress, true); - } else { - assert.strictEqual(params.replayId, testUploadId); + const origPutObject = metadataBackend.putObject; + metadataBackend.putObject = (putBucketName, objName, objVal, params, log, cb) => { + assert.strictEqual(params.oldReplayId, testUploadId); metadataBackend.putObject = origPutObject; - } - origPutObject( - putBucketName, objName, objVal, params, log, cb); - callCount += 1; - }; - const parts = [{ partNumber: 1, eTag }]; - const completeRequest = _createCompleteMpuRequest(testUploadId, - parts); - completeMultipartUpload(authInfo, completeRequest, log, - err => next(err, testUploadId)); - }, - (testUploadId, next) => { - versioningTestUtils.assertDataStoreValues( - ds, [undefined, objData[1], objData[2]]); - - const origPutObject = metadataBackend.putObject; - metadataBackend.putObject = - (putBucketName, objName, objVal, params, log, cb) => { - assert.strictEqual(params.oldReplayId, testUploadId); - metadataBackend.putObject = origPutObject; - origPutObject( - putBucketName, objName, objVal, params, log, cb); - }; - // overwrite null version with a non-MPU object - objectPut(authInfo, testPutObjectRequests[1], - undefined, log, err => next(err)); - }, - ], err => { - assert.ifError(err, `Unexpected err: ${err}`); - done(); - }); - }); + origPutObject(putBucketName, objName, objVal, params, log, cb); + }; + // overwrite null version with a non-MPU object + objectPut(authInfo, testPutObjectRequests[1], undefined, log, err => next(err)); + }, + ], + err => { + assert.ifError(err, `Unexpected err: ${err}`); + done(); + }, + ); + }, + ); - it('should finish deleting metadata on completeMultipartUpload retry', - done => { + it('should finish deleting metadata on completeMultipartUpload retry', done => { let origDeleteObject; - async.waterfall([ - next => bucketPutVersioning(authInfo, - enableVersioningRequest, log, err => next(err)), - next => - initiateMultipartUpload(authInfo, initiateRequest, log, next), - (result, corsHeaders, next) => parseString(result, next), - (json, next) => { - const partBody = objData[2]; - const testUploadId = - json.InitiateMultipartUploadResult.UploadId[0]; - const partRequest = _createPutPartRequest(testUploadId, 1, - partBody); - objectPutPart(authInfo, partRequest, undefined, log, - (err, eTag) => next(err, eTag, testUploadId)); - }, - (eTag, testUploadId, next) => { - origDeleteObject = metadataBackend.deleteObject; - metadataBackend.deleteObject = ( - bucketName, objName, params, log, cb) => { + async.waterfall( + [ + next => bucketPutVersioning(authInfo, enableVersioningRequest, log, err => next(err)), + next => initiateMultipartUpload(authInfo, initiateRequest, log, next), + (result, corsHeaders, next) => parseString(result, next), + (json, next) => { + const partBody = objData[2]; + const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; + const partRequest = _createPutPartRequest(testUploadId, 1, partBody); + objectPutPart(authInfo, partRequest, undefined, log, (err, eTag) => next(err, eTag, testUploadId)); + }, + (eTag, testUploadId, next) => { + origDeleteObject = metadataBackend.deleteObject; + metadataBackend.deleteObject = (bucketName, objName, params, log, cb) => { // prevent deletions from MPU bucket only - if (bucketName === mpuBucket) { - return process.nextTick( - () => cb(errors.InternalError)); + if (bucketName === mpuBucket) { + return process.nextTick(() => cb(errors.InternalError)); + } + return origDeleteObject(bucketName, objName, params, log, cb); + }; + const parts = [{ partNumber: 1, eTag }]; + const completeRequest = _createCompleteMpuRequest(testUploadId, parts); + completeMultipartUpload(authInfo, completeRequest, log, err => { + // expect a failure here because we could not + // remove the overview key + assert.strictEqual(err.is.InternalError, true); + next(null, eTag, testUploadId); + }); + }, + (eTag, testUploadId, next) => { + // allow MPU bucket metadata deletions to happen again + metadataBackend.deleteObject = origDeleteObject; + // retry the completeMultipartUpload with the same + // metadata, as an application would normally do after + // a failure + const parts = [{ partNumber: 1, eTag }]; + const completeRequest = _createCompleteMpuRequest(testUploadId, parts); + completeMultipartUpload(authInfo, completeRequest, log, next); + }, + ], + err => { + assert.ifError(err); + let nbVersions = 0; + for (const key of metadata.keyMaps.get(bucketName).keys()) { + if (key !== objectKey && key.startsWith(objectKey)) { + nbVersions += 1; } - return origDeleteObject( - bucketName, objName, params, log, cb); - }; - const parts = [{ partNumber: 1, eTag }]; - const completeRequest = _createCompleteMpuRequest( - testUploadId, parts); - completeMultipartUpload(authInfo, completeRequest, log, err => { - // expect a failure here because we could not - // remove the overview key - assert.strictEqual(err.is.InternalError, true); - next(null, eTag, testUploadId); - }); - }, - (eTag, testUploadId, next) => { - // allow MPU bucket metadata deletions to happen again - metadataBackend.deleteObject = origDeleteObject; - // retry the completeMultipartUpload with the same - // metadata, as an application would normally do after - // a failure - const parts = [{ partNumber: 1, eTag }]; - const completeRequest = _createCompleteMpuRequest( - testUploadId, parts); - completeMultipartUpload(authInfo, completeRequest, log, next); - }, - ], err => { - assert.ifError(err); - let nbVersions = 0; - for (const key of metadata.keyMaps.get(bucketName).keys()) { - if (key !== objectKey && key.startsWith(objectKey)) { - nbVersions += 1; } - } - // There should be only one version of the object, since - // the second call should not have created a new version - assert.strictEqual(nbVersions, 1); - for (const key of metadata.keyMaps.get(mpuBucket).keys()) { - assert.fail('There should be no more keys in MPU bucket, ' + - `found "${key}"`); - } - done(); - }); + // There should be only one version of the object, since + // the second call should not have created a new version + assert.strictEqual(nbVersions, 1); + for (const key of metadata.keyMaps.get(mpuBucket).keys()) { + assert.fail('There should be no more keys in MPU bucket, ' + `found "${key}"`); + } + done(); + }, + ); }); - it('should complete an MPU and promote its MD if it has been created by a failed complete before' + - 'without creating a new version', done => { - const delMeta = metadataBackend.deleteObject; - metadataBackend.deleteObject = (bucketName, objName, params, log, cb) => cb(errors.InternalError); - const partBody = Buffer.from('I am a part\n', 'utf8'); - initiateRequest.headers['x-amz-meta-stuff'] = - 'I am some user metadata'; - async.waterfall([ - next => bucketPutVersioning(authInfo, - enableVersioningRequest, log, err => next(err)), - next => initiateMultipartUpload(authInfo, - initiateRequest, log, next), - (result, corsHeaders, next) => parseString(result, next), - ], - (err, json) => { - assert.ifError(err); - const testUploadId = - json.InitiateMultipartUploadResult.UploadId[0]; - const partHash = crypto.createHash('md5').update(partBody).digest('hex'); - const partRequest = new DummyRequest({ - bucketName, - namespace, - objectKey, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, - query: { - partNumber: '1', - uploadId: testUploadId, - }, - partHash, - }, partBody); - objectPutPart(authInfo, partRequest, undefined, log, () => { - const completeBody = '' + - '' + - '1' + - `"${partHash}"` + - '' + - ''; - const completeRequest = { - bucketName, - namespace, - objectKey, - parsedHost: 's3.amazonaws.com', - url: `/${objectKey}?uploadId=${testUploadId}`, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - query: { uploadId: testUploadId }, - post: completeBody, - actionImplicitDenies: false, - }; - completeMultipartUpload(authInfo, - completeRequest, log, err => { - assert(err.is.InternalError); - const MD = metadata.keyMaps.get(bucketName) - .get(objectKey); - assert(MD); - const firstVersionId = MD.versionId; - assert.strictEqual(MD['x-amz-meta-stuff'], - 'I am some user metadata'); - assert.strictEqual(MD.uploadId, testUploadId); - metadataBackend.deleteObject = delMeta; - assert.strictEqual(metadata.keyMaps.get(mpuBucket).size, 2); - completeMultipartUpload(authInfo, - completeRequest, log, err => { + it( + 'should complete an MPU and promote its MD if it has been created by a failed complete before' + + 'without creating a new version', + done => { + const delMeta = metadataBackend.deleteObject; + metadataBackend.deleteObject = (bucketName, objName, params, log, cb) => cb(errors.InternalError); + const partBody = Buffer.from('I am a part\n', 'utf8'); + initiateRequest.headers['x-amz-meta-stuff'] = 'I am some user metadata'; + async.waterfall( + [ + next => bucketPutVersioning(authInfo, enableVersioningRequest, log, err => next(err)), + next => initiateMultipartUpload(authInfo, initiateRequest, log, next), + (result, corsHeaders, next) => parseString(result, next), + ], + (err, json) => { + assert.ifError(err); + const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; + const partHash = crypto.createHash('md5').update(partBody).digest('hex'); + const partRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, + query: { + partNumber: '1', + uploadId: testUploadId, + }, + partHash, + }, + partBody, + ); + objectPutPart(authInfo, partRequest, undefined, log, () => { + const completeBody = + '' + + '' + + '1' + + `"${partHash}"` + + '' + + ''; + const completeRequest = { + bucketName, + namespace, + objectKey, + parsedHost: 's3.amazonaws.com', + url: `/${objectKey}?uploadId=${testUploadId}`, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + query: { uploadId: testUploadId }, + post: completeBody, + actionImplicitDenies: false, + }; + completeMultipartUpload(authInfo, completeRequest, log, err => { + assert(err.is.InternalError); + const MD = metadata.keyMaps.get(bucketName).get(objectKey); + assert(MD); + const firstVersionId = MD.versionId; + assert.strictEqual(MD['x-amz-meta-stuff'], 'I am some user metadata'); + assert.strictEqual(MD.uploadId, testUploadId); + metadataBackend.deleteObject = delMeta; + assert.strictEqual(metadata.keyMaps.get(mpuBucket).size, 2); + completeMultipartUpload(authInfo, completeRequest, log, err => { assert.ifError(err); - const MD = metadata.keyMaps.get(bucketName) - .get(objectKey); + const MD = metadata.keyMaps.get(bucketName).get(objectKey); assert(MD); assert.strictEqual(MD.versionId, firstVersionId); - assert.strictEqual(MD['x-amz-meta-stuff'], - 'I am some user metadata'); + assert.strictEqual(MD['x-amz-meta-stuff'], 'I am some user metadata'); assert.strictEqual(metadata.keyMaps.get(mpuBucket).size, 0); done(); }); + }); }); - }); - }); - }); + }, + ); + }, + ); }); describe('multipart upload with object lock', () => { @@ -2691,82 +2758,74 @@ describe('multipart upload with object lock', () => { after(cleanup); - it('mpu object should contain retention info when mpu initiated with ' + - 'object retention', done => { + it('mpu object should contain retention info when mpu initiated with ' + 'object retention', done => { let versionId; - async.waterfall([ - next => initiateMultipartUpload(authInfo, retentionInitiateRequest, - log, next), - (result, corsHeaders, next) => parseString(result, next), - (json, next) => { - const partBody = Buffer.from('foobar', 'utf8'); - const testUploadId = - json.InitiateMultipartUploadResult.UploadId[0]; - const partRequest = _createPutPartRequest(testUploadId, 1, - partBody); - partRequest.bucketName = lockedBucket; - partRequest.headers = { host: `${lockedBucket}.s3.amazonaws.com` }; - objectPutPart(authInfo, partRequest, undefined, log, - (err, eTag) => next(err, eTag, testUploadId)); - }, - (eTag, testUploadId, next) => { - const parts = [{ partNumber: 1, eTag }]; - const completeRequest = _createCompleteMpuRequest(testUploadId, - parts); - completeRequest.bucketName = lockedBucket; - completeRequest.headers = { host: `${lockedBucket}.s3.amazonaws.com` }; - completeMultipartUpload(authInfo, completeRequest, log, next); - }, - (xml, headers, next) => { - versionId = headers['x-amz-version-id']; - getObjectRetention(authInfo, getObjectLockInfoRequest, log, next); + async.waterfall( + [ + next => initiateMultipartUpload(authInfo, retentionInitiateRequest, log, next), + (result, corsHeaders, next) => parseString(result, next), + (json, next) => { + const partBody = Buffer.from('foobar', 'utf8'); + const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; + const partRequest = _createPutPartRequest(testUploadId, 1, partBody); + partRequest.bucketName = lockedBucket; + partRequest.headers = { host: `${lockedBucket}.s3.amazonaws.com` }; + objectPutPart(authInfo, partRequest, undefined, log, (err, eTag) => next(err, eTag, testUploadId)); + }, + (eTag, testUploadId, next) => { + const parts = [{ partNumber: 1, eTag }]; + const completeRequest = _createCompleteMpuRequest(testUploadId, parts); + completeRequest.bucketName = lockedBucket; + completeRequest.headers = { host: `${lockedBucket}.s3.amazonaws.com` }; + completeMultipartUpload(authInfo, completeRequest, log, next); + }, + (xml, headers, next) => { + versionId = headers['x-amz-version-id']; + getObjectRetention(authInfo, getObjectLockInfoRequest, log, next); + }, + (result, corsHeaders, next) => parseString(result, next), + ], + (err, json) => { + assert.ifError(err); + assert.deepStrictEqual(json.Retention, expectedRetentionConfig); + changeObjectLock([{ bucket: lockedBucket, key: objectKey, versionId }], '', done); }, - (result, corsHeaders, next) => parseString(result, next), - ], (err, json) => { - assert.ifError(err); - assert.deepStrictEqual(json.Retention, expectedRetentionConfig); - changeObjectLock( - [{ bucket: lockedBucket, key: objectKey, versionId }], '', done); - }); + ); }); - it('mpu object should contain legal hold info when mpu initiated with ' + - 'legal hold', done => { + it('mpu object should contain legal hold info when mpu initiated with ' + 'legal hold', done => { let versionId; - async.waterfall([ - next => initiateMultipartUpload(authInfo, legalHoldInitiateRequest, - log, next), - (result, corsHeaders, next) => parseString(result, next), - (json, next) => { - const partBody = Buffer.from('foobar', 'utf8'); - const testUploadId = - json.InitiateMultipartUploadResult.UploadId[0]; - const partRequest = _createPutPartRequest(testUploadId, 1, - partBody); - partRequest.bucketName = lockedBucket; - partRequest.headers = { host: `${lockedBucket}.s3.amazonaws.com` }; - objectPutPart(authInfo, partRequest, undefined, log, - (err, eTag) => next(err, eTag, testUploadId)); - }, - (eTag, testUploadId, next) => { - const parts = [{ partNumber: 1, eTag }]; - const completeRequest = _createCompleteMpuRequest(testUploadId, - parts); - completeRequest.bucketName = lockedBucket; - completeRequest.headers = { host: `${lockedBucket}.s3.amazonaws.com` }; - completeMultipartUpload(authInfo, completeRequest, log, next); - }, - (xml, headers, next) => { - versionId = headers['x-amz-version-id']; - getObjectLegalHold(authInfo, getObjectLockInfoRequest, log, next); + async.waterfall( + [ + next => initiateMultipartUpload(authInfo, legalHoldInitiateRequest, log, next), + (result, corsHeaders, next) => parseString(result, next), + (json, next) => { + const partBody = Buffer.from('foobar', 'utf8'); + const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; + const partRequest = _createPutPartRequest(testUploadId, 1, partBody); + partRequest.bucketName = lockedBucket; + partRequest.headers = { host: `${lockedBucket}.s3.amazonaws.com` }; + objectPutPart(authInfo, partRequest, undefined, log, (err, eTag) => next(err, eTag, testUploadId)); + }, + (eTag, testUploadId, next) => { + const parts = [{ partNumber: 1, eTag }]; + const completeRequest = _createCompleteMpuRequest(testUploadId, parts); + completeRequest.bucketName = lockedBucket; + completeRequest.headers = { host: `${lockedBucket}.s3.amazonaws.com` }; + completeMultipartUpload(authInfo, completeRequest, log, next); + }, + (xml, headers, next) => { + versionId = headers['x-amz-version-id']; + getObjectLegalHold(authInfo, getObjectLockInfoRequest, log, next); + }, + (result, corsHeaders, next) => parseString(result, next), + ], + (err, json) => { + assert.ifError(err); + assert.deepStrictEqual(json.LegalHold, expectedLegalHold); + changeObjectLock([{ bucket: lockedBucket, key: objectKey, versionId }], '', done); }, - (result, corsHeaders, next) => parseString(result, next), - ], (err, json) => { - assert.ifError(err); - assert.deepStrictEqual(json.LegalHold, expectedLegalHold); - changeObjectLock( - [{ bucket: lockedBucket, key: objectKey, versionId }], '', done); - }); + ); }); }); @@ -2784,46 +2843,56 @@ describe('multipart upload overheadField', () => { }); it('should pass overheadField', done => { - async.waterfall([ - next => bucketPut(authInfo, bucketPutRequest, log, next), - (corsHeaders, next) => initiateMultipartUpload(authInfo, - initiateRequest, log, next), - (result, corsHeaders, next) => { - const mpuKeys = metadata.keyMaps.get(mpuBucket); - assert.strictEqual(mpuKeys.size, 1); - assert(mpuKeys.keys().next().value - .startsWith(`overview${splitter}${objectKey}`)); - parseString(result, next); - }, - ], - (err, json) => { - // Need to build request in here since do not have uploadId - // until here - assert.ifError(err); - const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; - const md5Hash = crypto.createHash('md5'); - const bufferBody = Buffer.from(postBody); - md5Hash.update(bufferBody); - const partHash = md5Hash.digest('hex'); - const partRequest = new DummyRequest({ - bucketName, - objectKey, - namespace, - url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - query: { - partNumber: '1', - uploadId: testUploadId, + async.waterfall( + [ + next => bucketPut(authInfo, bucketPutRequest, log, next), + (corsHeaders, next) => initiateMultipartUpload(authInfo, initiateRequest, log, next), + (result, corsHeaders, next) => { + const mpuKeys = metadata.keyMaps.get(mpuBucket); + assert.strictEqual(mpuKeys.size, 1); + assert(mpuKeys.keys().next().value.startsWith(`overview${splitter}${objectKey}`)); + parseString(result, next); }, - partHash, - }, postBody); - objectPutPart(authInfo, partRequest, undefined, log, err => { + ], + (err, json) => { + // Need to build request in here since do not have uploadId + // until here assert.ifError(err); - sinon.assert.calledWith(metadataswitch.putObjectMD.lastCall, - any, any, any, sinon.match({ overheadField: sinon.match.array }), any, any); - done(); - }); - }); + const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; + const md5Hash = crypto.createHash('md5'); + const bufferBody = Buffer.from(postBody); + md5Hash.update(bufferBody); + const partHash = md5Hash.digest('hex'); + const partRequest = new DummyRequest( + { + bucketName, + objectKey, + namespace, + url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + query: { + partNumber: '1', + uploadId: testUploadId, + }, + partHash, + }, + postBody, + ); + objectPutPart(authInfo, partRequest, undefined, log, err => { + assert.ifError(err); + sinon.assert.calledWith( + metadataswitch.putObjectMD.lastCall, + any, + any, + any, + sinon.match({ overheadField: sinon.match.array }), + any, + any, + ); + done(); + }); + }, + ); }); }); @@ -2850,12 +2919,13 @@ describe('complete mpu with bucket policy', () => { const partBody = Buffer.from('I am a part\n', 'utf8'); const md5Hash = crypto.createHash('md5').update(partBody); const partHash = md5Hash.digest('hex'); - const completeBody = '' + - '' + - '1' + - `"${partHash}"` + - '' + - ''; + const completeBody = + '' + + '' + + '1' + + `"${partHash}"` + + '' + + ''; beforeEach(done => { cleanup(); @@ -2882,56 +2952,64 @@ describe('complete mpu with bucket policy', () => { /** root user doesn't check bucket policy */ const authNotRoot = makeAuthInfo(canonicalID, 'not-root'); - async.waterfall([ - next => bucketPutPolicy(authInfo, - bucketPutPolicyRequest, log, next), - (corsHeaders, next) => initiateMultipartUpload(authNotRoot, - initiateReqFixed, log, next), - (result, corsHeaders, next) => parseString(result, next), - (json, next) => { - const testUploadId = - json.InitiateMultipartUploadResult.UploadId[0]; - const partRequest = new DummyRequest(Object.assign({ - socket: { - remoteAddress: '1.1.1.1', - }, - bucketName, - namespace, - objectKey, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, - query: { - partNumber: '1', - uploadId: testUploadId, - }, - partHash, - }, requestFix), partBody); - objectPutPart(authNotRoot, partRequest, - undefined, log, err => next(err, testUploadId)); - }, - (testUploadId, next) => { - const completeRequest = new DummyRequest(Object.assign({ - socket: { - remoteAddress: '1.1.1.1', - }, - bucketName, - namespace, - objectKey, - parsedHost: 's3.amazonaws.com', - url: `/${objectKey}?uploadId=${testUploadId}`, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - query: { uploadId: testUploadId }, - post: completeBody, - actionImplicitDenies: false, - }, requestFix)); - completeMultipartUpload(authNotRoot, completeRequest, - log, next); + async.waterfall( + [ + next => bucketPutPolicy(authInfo, bucketPutPolicyRequest, log, next), + (corsHeaders, next) => initiateMultipartUpload(authNotRoot, initiateReqFixed, log, next), + (result, corsHeaders, next) => parseString(result, next), + (json, next) => { + const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; + const partRequest = new DummyRequest( + Object.assign( + { + socket: { + remoteAddress: '1.1.1.1', + }, + bucketName, + namespace, + objectKey, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, + query: { + partNumber: '1', + uploadId: testUploadId, + }, + partHash, + }, + requestFix, + ), + partBody, + ); + objectPutPart(authNotRoot, partRequest, undefined, log, err => next(err, testUploadId)); + }, + (testUploadId, next) => { + const completeRequest = new DummyRequest( + Object.assign( + { + socket: { + remoteAddress: '1.1.1.1', + }, + bucketName, + namespace, + objectKey, + parsedHost: 's3.amazonaws.com', + url: `/${objectKey}?uploadId=${testUploadId}`, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + query: { uploadId: testUploadId }, + post: completeBody, + actionImplicitDenies: false, + }, + requestFix, + ), + ); + completeMultipartUpload(authNotRoot, completeRequest, log, next); + }, + ], + err => { + assert.ifError(err); + done(); }, - ], - err => { - assert.ifError(err); - done(); - }); + ); }); it('should set bucketOwnerId if requester is not destination bucket owner', done => { @@ -2947,66 +3025,76 @@ describe('complete mpu with bucket policy', () => { }, ], }); - async.waterfall([ - next => bucketPutPolicy(authInfo, bucketPutPolicyRequest, log, next), - (corsHeaders, next) => initiateMultipartUpload(authInfoOtherAcc, - initiateRequest, log, next), - (result, corsHeaders, next) => parseString(result, next), - ], - (err, json) => { - // Need to build request in here since do not have uploadId - // until here - assert.ifError(err); - const testUploadId = - json.InitiateMultipartUploadResult.UploadId[0]; - const partHash = crypto.createHash('md5').update(partBody).digest('hex'); - const partRequest = new DummyRequest(Object.assign({ - bucketName, - namespace, - objectKey, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, - query: { - partNumber: '1', - uploadId: testUploadId, - }, - // Note that the body of the post set in the request here does - // not really matter in this test. - // The put is not going through the route so the md5 is being - // calculated above and manually being set in the request below. - // What is being tested is that the partHash being sent - // to the API for the part is stored and then used to - // calculate the final ETag upon completion - // of the multipart upload. - partHash, - socket: { - remoteAddress: '1.1.1.1', - }, - }, requestFix), partBody); - objectPutPart(authInfoOtherAcc, partRequest, undefined, log, err => { + async.waterfall( + [ + next => bucketPutPolicy(authInfo, bucketPutPolicyRequest, log, next), + (corsHeaders, next) => initiateMultipartUpload(authInfoOtherAcc, initiateRequest, log, next), + (result, corsHeaders, next) => parseString(result, next), + ], + (err, json) => { + // Need to build request in here since do not have uploadId + // until here assert.ifError(err); - const completeBody = '' + - '' + - '1' + - `"${partHash}"` + - '' + - ''; - const completeRequest = new DummyRequest(Object.assign({ - bucketName, - namespace, - objectKey, - parsedHost: 's3.amazonaws.com', - url: `/${objectKey}?uploadId=${testUploadId}`, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - query: { uploadId: testUploadId }, - post: completeBody, - actionImplicitDenies: false, - socket: { - remoteAddress: '1.1.1.1', - }, - }, requestFix)); - completeMultipartUpload(authInfoOtherAcc, - completeRequest, log, err => { + const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; + const partHash = crypto.createHash('md5').update(partBody).digest('hex'); + const partRequest = new DummyRequest( + Object.assign( + { + bucketName, + namespace, + objectKey, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: `/${objectKey}?partNumber=1&uploadId=${testUploadId}`, + query: { + partNumber: '1', + uploadId: testUploadId, + }, + // Note that the body of the post set in the request here does + // not really matter in this test. + // The put is not going through the route so the md5 is being + // calculated above and manually being set in the request below. + // What is being tested is that the partHash being sent + // to the API for the part is stored and then used to + // calculate the final ETag upon completion + // of the multipart upload. + partHash, + socket: { + remoteAddress: '1.1.1.1', + }, + }, + requestFix, + ), + partBody, + ); + objectPutPart(authInfoOtherAcc, partRequest, undefined, log, err => { + assert.ifError(err); + const completeBody = + '' + + '' + + '1' + + `"${partHash}"` + + '' + + ''; + const completeRequest = new DummyRequest( + Object.assign( + { + bucketName, + namespace, + objectKey, + parsedHost: 's3.amazonaws.com', + url: `/${objectKey}?uploadId=${testUploadId}`, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + query: { uploadId: testUploadId }, + post: completeBody, + actionImplicitDenies: false, + socket: { + remoteAddress: '1.1.1.1', + }, + }, + requestFix, + ), + ); + completeMultipartUpload(authInfoOtherAcc, completeRequest, log, err => { assert.ifError(err); sinon.assert.calledWith( metadataswitch.putObjectMD.lastCall, @@ -3015,12 +3103,13 @@ describe('complete mpu with bucket policy', () => { sinon.match({ bucketOwnerId: authInfo.canonicalId }), sinon.match.any, sinon.match.any, - sinon.match.any + sinon.match.any, ); done(); }); - }); - }); + }); + }, + ); }); }); @@ -3035,10 +3124,16 @@ describe('multipart upload in ingestion bucket', () => { versionID = versioning.VersionID.encode(versioning.VersionID.generateVersionId('0', '')); // Setup multi-backend, this is required for ingestion - data.switch(new storage.data.MultipleBackendGateway({ - 'us-east-1': dataClient, - 'us-east-2': dataClient, - }, metadata, data.locStorageCheckFn)); + data.switch( + new storage.data.MultipleBackendGateway( + { + 'us-east-1': dataClient, + 'us-east-2': dataClient, + }, + metadata, + data.locStorageCheckFn, + ), + ); data.implName = 'multipleBackends'; // "mock" the data location, simulating a backend supporting MPU @@ -3076,17 +3171,19 @@ describe('multipart upload in ingestion bucket', () => { sinon.restore(); }); - const newPutIngestBucketRequest = location => new DummyRequest({ - bucketName, - namespace, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - url: '/', - post: '' + - '' + - `${location}` + - '', - }); + const newPutIngestBucketRequest = location => + new DummyRequest({ + bucketName, + namespace, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: '/', + post: + '' + + '' + + `${location}` + + '', + }); const archiveRestoreRequested = { archiveInfo: { foo: 0, bar: 'stuff' }, // opaque, can be anything... restoreRequestedAt: new Date().toString(), @@ -3184,14 +3281,15 @@ describe('initiateMultipartUpload with objectKeyByteLimit', () => { config.objectKeyByteLimit = originalObjectKeyByteLimit; }); - const createTestInitiateRequest = longKey => new DummyRequest({ - bucketName, - namespace, - objectKey: longKey, - headers: {}, - url: `/${bucketName}/${longKey}?uploads`, - query: { uploads: '' }, - }); + const createTestInitiateRequest = longKey => + new DummyRequest({ + bucketName, + namespace, + objectKey: longKey, + headers: {}, + url: `/${bucketName}/${longKey}?uploads`, + query: { uploads: '' }, + }); it('should reject object key longer than 915 bytes by default', done => { const longKey = 'a'.repeat(916); diff --git a/tests/unit/api/objectACLauth.js b/tests/unit/api/objectACLauth.js index df0ffad322..3edc896581 100644 --- a/tests/unit/api/objectACLauth.js +++ b/tests/unit/api/objectACLauth.js @@ -2,8 +2,7 @@ const assert = require('assert'); const BucketInfo = require('arsenal').models.BucketInfo; const constants = require('../../../constants'); -const { isObjAuthorized } - = require('../../../lib/api/apiUtils/authorization/permissionChecks'); +const { isObjAuthorized } = require('../../../lib/api/apiUtils/authorization/permissionChecks'); const { DummyRequestLogger, makeAuthInfo } = require('../helpers'); const accessKey = 'accessKey1'; @@ -15,12 +14,11 @@ const userAuthInfo = makeAuthInfo(accessKey, 'user'); const altAcctAuthInfo = makeAuthInfo(altAccessKey); const accountToVet = altAcctAuthInfo.getCanonicalID(); -const bucket = new BucketInfo('niftyBucket', bucketOwnerCanonicalId, - 'iAmTheOwnerDisplayName', creationDate); +const bucket = new BucketInfo('niftyBucket', bucketOwnerCanonicalId, 'iAmTheOwnerDisplayName', creationDate); const objectOwnerCanonicalId = userAuthInfo.getCanonicalID(); const object = { 'owner-id': objectOwnerCanonicalId, - 'acl': { + acl: { Canned: 'private', FULL_CONTROL: [], WRITE_ACP: [], @@ -46,60 +44,61 @@ describe('object acl authorization for objectGet and objectHead', () => { it('should allow access to object owner', () => { const results = requestTypes.map(type => - isObjAuthorized(bucket, object, type, objectOwnerCanonicalId, - authInfo, log)); + isObjAuthorized(bucket, object, type, objectOwnerCanonicalId, authInfo, log), + ); assert.deepStrictEqual(results, [true, true]); }); it('should allow access to user in object owner account', () => { const results = requestTypes.map(type => - isObjAuthorized(bucket, object, type, objectOwnerCanonicalId, - userAuthInfo, log)); + isObjAuthorized(bucket, object, type, objectOwnerCanonicalId, userAuthInfo, log), + ); assert.deepStrictEqual(results, [true, true]); }); it('should allow access to bucket owner if same account as object owner', () => { const results = requestTypes.map(type => - isObjAuthorized(bucket, object, type, bucketOwnerCanonicalId, - authInfo, log)); + isObjAuthorized(bucket, object, type, bucketOwnerCanonicalId, authInfo, log), + ); assert.deepStrictEqual(results, [true, true]); }); it('should allow access to anyone if canned public-read ACL', () => { object.acl.Canned = 'public-read'; const results = requestTypes.map(type => - isObjAuthorized(bucket, object, type, accountToVet, altAcctAuthInfo, log)); + isObjAuthorized(bucket, object, type, accountToVet, altAcctAuthInfo, log), + ); assert.deepStrictEqual(results, [true, true]); }); it('should allow access to anyone if canned public-read-write ACL', () => { object.acl.Canned = 'public-read-write'; const results = requestTypes.map(type => - isObjAuthorized(bucket, object, type, accountToVet, altAcctAuthInfo, log)); + isObjAuthorized(bucket, object, type, accountToVet, altAcctAuthInfo, log), + ); assert.deepStrictEqual(results, [true, true]); }); - it('should not allow access to public user if ' + - 'authenticated-read ACL', () => { + it('should not allow access to public user if ' + 'authenticated-read ACL', () => { object.acl.Canned = 'authenticated-read'; const publicResults = requestTypes.map(type => - isObjAuthorized(bucket, object, type, constants.publicId, null, log)); + isObjAuthorized(bucket, object, type, constants.publicId, null, log), + ); assert.deepStrictEqual(publicResults, [false, false]); }); - it('should allow access to any authenticated user if ' + - 'authenticated-read ACL', () => { + it('should allow access to any authenticated user if ' + 'authenticated-read ACL', () => { object.acl.Canned = 'authenticated-read'; const results = requestTypes.map(type => - isObjAuthorized(bucket, object, type, accountToVet, altAcctAuthInfo, log)); + isObjAuthorized(bucket, object, type, accountToVet, altAcctAuthInfo, log), + ); assert.deepStrictEqual(results, [true, true]); }); - it('should allow access to bucket owner when object owner is alt account if ' + - 'bucket-owner-read ACL', () => { + it('should allow access to bucket owner when object owner is alt account if ' + 'bucket-owner-read ACL', () => { const altAcctObj = { 'owner-id': accountToVet, - 'acl': { + acl: { Canned: 'private', FULL_CONTROL: [], WRITE_ACP: [], @@ -108,70 +107,76 @@ describe('object acl authorization for objectGet and objectHead', () => { }, }; const noAuthResults = requestTypes.map(type => - isObjAuthorized(bucket, altAcctObj, type, bucketOwnerCanonicalId, authInfo, - log)); + isObjAuthorized(bucket, altAcctObj, type, bucketOwnerCanonicalId, authInfo, log), + ); assert.deepStrictEqual(noAuthResults, [false, false]); altAcctObj.acl.Canned = 'bucket-owner-read'; const authResults = requestTypes.map(type => - isObjAuthorized(bucket, altAcctObj, type, bucketOwnerCanonicalId, authInfo, - log)); + isObjAuthorized(bucket, altAcctObj, type, bucketOwnerCanonicalId, authInfo, log), + ); assert.deepStrictEqual(authResults, [true, true]); }); - it('should allow access to bucket owner when object owner is alt account if ' + - 'bucket-owner-full-control ACL', () => { - const altAcctObj = { - 'owner-id': accountToVet, - 'acl': { - Canned: 'private', - FULL_CONTROL: [], - WRITE_ACP: [], - READ: [], - READ_ACP: [], - }, - }; - const noAuthResults = requestTypes.map(type => - isObjAuthorized(bucket, altAcctObj, type, bucketOwnerCanonicalId, authInfo, - log)); - assert.deepStrictEqual(noAuthResults, [false, false]); - altAcctObj.acl.Canned = 'bucket-owner-full-control'; - const authResults = requestTypes.map(type => - isObjAuthorized(bucket, altAcctObj, type, bucketOwnerCanonicalId, authInfo, - log)); - assert.deepStrictEqual(authResults, [true, true]); - }); + it( + 'should allow access to bucket owner when object owner is alt account if ' + 'bucket-owner-full-control ACL', + () => { + const altAcctObj = { + 'owner-id': accountToVet, + acl: { + Canned: 'private', + FULL_CONTROL: [], + WRITE_ACP: [], + READ: [], + READ_ACP: [], + }, + }; + const noAuthResults = requestTypes.map(type => + isObjAuthorized(bucket, altAcctObj, type, bucketOwnerCanonicalId, authInfo, log), + ); + assert.deepStrictEqual(noAuthResults, [false, false]); + altAcctObj.acl.Canned = 'bucket-owner-full-control'; + const authResults = requestTypes.map(type => + isObjAuthorized(bucket, altAcctObj, type, bucketOwnerCanonicalId, authInfo, log), + ); + assert.deepStrictEqual(authResults, [true, true]); + }, + ); - it('should allow access to account if ' + - 'account was granted FULL_CONTROL', () => { + it('should allow access to account if ' + 'account was granted FULL_CONTROL', () => { const noAuthResults = requestTypes.map(type => - isObjAuthorized(bucket, object, type, accountToVet, altAcctAuthInfo, log)); + isObjAuthorized(bucket, object, type, accountToVet, altAcctAuthInfo, log), + ); assert.deepStrictEqual(noAuthResults, [false, false]); object.acl.FULL_CONTROL = [accountToVet]; const authResults = requestTypes.map(type => - isObjAuthorized(bucket, object, type, accountToVet, altAcctAuthInfo, log)); + isObjAuthorized(bucket, object, type, accountToVet, altAcctAuthInfo, log), + ); assert.deepStrictEqual(authResults, [true, true]); }); - it('should allow access to account if ' + - 'account was granted READ right', () => { + it('should allow access to account if ' + 'account was granted READ right', () => { const noAuthResults = requestTypes.map(type => - isObjAuthorized(bucket, object, type, accountToVet, altAcctAuthInfo, log)); + isObjAuthorized(bucket, object, type, accountToVet, altAcctAuthInfo, log), + ); assert.deepStrictEqual(noAuthResults, [false, false]); object.acl.READ = [accountToVet]; const authResults = requestTypes.map(type => - isObjAuthorized(bucket, object, type, accountToVet, altAcctAuthInfo, log)); + isObjAuthorized(bucket, object, type, accountToVet, altAcctAuthInfo, log), + ); assert.deepStrictEqual(authResults, [true, true]); }); it('should not allow access to public user if private canned ACL', () => { const results = requestTypes.map(type => - isObjAuthorized(bucket, object, type, accountToVet, altAcctAuthInfo, log)); + isObjAuthorized(bucket, object, type, accountToVet, altAcctAuthInfo, log), + ); assert.deepStrictEqual(results, [false, false]); }); it('should not allow access to just any user if private canned ACL', () => { const results = requestTypes.map(type => - isObjAuthorized(bucket, object, type, accountToVet, altAcctAuthInfo, log)); + isObjAuthorized(bucket, object, type, accountToVet, altAcctAuthInfo, log), + ); assert.deepStrictEqual(results, [false, false]); }); }); @@ -180,11 +185,12 @@ describe('object authorization for objectPut and objectDelete', () => { it('should allow access when no implicitDeny information is provided', () => { const requestTypes = ['objectPut', 'objectDelete']; const results = requestTypes.map(type => - isObjAuthorized(bucket, object, type, accountToVet, altAcctAuthInfo, log)); + isObjAuthorized(bucket, object, type, accountToVet, altAcctAuthInfo, log), + ); assert.deepStrictEqual(results, [true, true]); const publicUserResults = requestTypes.map(type => - isObjAuthorized(bucket, object, type, constants.publicId, null, - log)); + isObjAuthorized(bucket, object, type, constants.publicId, null, log), + ); assert.deepStrictEqual(publicUserResults, [true, true]); }); }); @@ -206,71 +212,69 @@ describe('object authorization for objectPutACL and objectGetACL', () => { it('should allow access to object owner', () => { const results = requestTypes.map(type => - isObjAuthorized(bucket, object, type, objectOwnerCanonicalId, - authInfo, log)); + isObjAuthorized(bucket, object, type, objectOwnerCanonicalId, authInfo, log), + ); assert.deepStrictEqual(results, [true, true]); }); it('should allow access to user in object owner account', () => { const results = requestTypes.map(type => - isObjAuthorized(bucket, object, type, objectOwnerCanonicalId, - userAuthInfo, log)); + isObjAuthorized(bucket, object, type, objectOwnerCanonicalId, userAuthInfo, log), + ); assert.deepStrictEqual(results, [true, true]); }); - it('should allow access to bucket owner when object owner is alt account if ' + - 'bucket-owner-full-control canned ACL set on object', () => { - const altAcctObj = { - 'owner-id': accountToVet, - 'acl': { - Canned: 'private', - FULL_CONTROL: [], - WRITE_ACP: [], - READ: [], - READ_ACP: [], - }, - }; - const noAuthResults = requestTypes.map(type => - isObjAuthorized(bucket, altAcctObj, type, bucketOwnerCanonicalId, authInfo, - log)); - assert.deepStrictEqual(noAuthResults, [false, false]); - altAcctObj.acl.Canned = 'bucket-owner-full-control'; - const authorizedResults = requestTypes.map(type => - isObjAuthorized(bucket, altAcctObj, type, bucketOwnerCanonicalId, authInfo, - null, log)); - assert.deepStrictEqual(authorizedResults, [true, true]); - }); + it( + 'should allow access to bucket owner when object owner is alt account if ' + + 'bucket-owner-full-control canned ACL set on object', + () => { + const altAcctObj = { + 'owner-id': accountToVet, + acl: { + Canned: 'private', + FULL_CONTROL: [], + WRITE_ACP: [], + READ: [], + READ_ACP: [], + }, + }; + const noAuthResults = requestTypes.map(type => + isObjAuthorized(bucket, altAcctObj, type, bucketOwnerCanonicalId, authInfo, log), + ); + assert.deepStrictEqual(noAuthResults, [false, false]); + altAcctObj.acl.Canned = 'bucket-owner-full-control'; + const authorizedResults = requestTypes.map(type => + isObjAuthorized(bucket, altAcctObj, type, bucketOwnerCanonicalId, authInfo, null, log), + ); + assert.deepStrictEqual(authorizedResults, [true, true]); + }, + ); - it('should allow access to account if ' + - 'account was granted FULL_CONTROL right', () => { + it('should allow access to account if ' + 'account was granted FULL_CONTROL right', () => { const noAuthResults = requestTypes.map(type => - isObjAuthorized(bucket, object, type, accountToVet, altAcctAuthInfo, log)); + isObjAuthorized(bucket, object, type, accountToVet, altAcctAuthInfo, log), + ); assert.deepStrictEqual(noAuthResults, [false, false]); object.acl.FULL_CONTROL = [accountToVet]; const authorizedResults = requestTypes.map(type => - isObjAuthorized(bucket, object, type, accountToVet, altAcctAuthInfo, log)); + isObjAuthorized(bucket, object, type, accountToVet, altAcctAuthInfo, log), + ); assert.deepStrictEqual(authorizedResults, [true, true]); }); - it('should allow objectPutACL access to account if ' + - 'account was granted WRITE_ACP right', () => { - const noAuthResult = isObjAuthorized(bucket, object, 'objectPutACL', - accountToVet, altAcctAuthInfo, log); + it('should allow objectPutACL access to account if ' + 'account was granted WRITE_ACP right', () => { + const noAuthResult = isObjAuthorized(bucket, object, 'objectPutACL', accountToVet, altAcctAuthInfo, log); assert.strictEqual(noAuthResult, false); object.acl.WRITE_ACP = [accountToVet]; - const authorizedResult = isObjAuthorized(bucket, object, 'objectPutACL', - accountToVet, altAcctAuthInfo, log); + const authorizedResult = isObjAuthorized(bucket, object, 'objectPutACL', accountToVet, altAcctAuthInfo, log); assert.strictEqual(authorizedResult, true); }); - it('should allow objectGetACL access to account if ' + - 'account was granted READ_ACP right', () => { - const noAuthResult = isObjAuthorized(bucket, object, 'objectGetACL', - accountToVet, altAcctAuthInfo, log); + it('should allow objectGetACL access to account if ' + 'account was granted READ_ACP right', () => { + const noAuthResult = isObjAuthorized(bucket, object, 'objectGetACL', accountToVet, altAcctAuthInfo, log); assert.strictEqual(noAuthResult, false); object.acl.READ_ACP = [accountToVet]; - const authorizedResult = isObjAuthorized(bucket, object, 'objectGetACL', - accountToVet, altAcctAuthInfo, log); + const authorizedResult = isObjAuthorized(bucket, object, 'objectGetACL', accountToVet, altAcctAuthInfo, log); assert.strictEqual(authorizedResult, true); }); }); @@ -288,12 +292,7 @@ describe('without object metadata', () => { bucket.setBucketPolicy(null); }); - const requestTypes = [ - 'objectGet', - 'objectHead', - 'objectPutACL', - 'objectGetACL', - ]; + const requestTypes = ['objectGet', 'objectHead', 'objectPutACL', 'objectGetACL']; const allowedAccess = [true, true, true, true]; const deniedAccess = [false, false, false, false]; @@ -301,77 +300,87 @@ describe('without object metadata', () => { const tests = [ { it: 'should allow user if part of the bucket owner account', - canned: 'private', id: objectOwnerCanonicalId, + canned: 'private', + id: objectOwnerCanonicalId, authInfo: userAuthInfo, aclParam: null, response: allowedAccess, }, { it: 'should not allow user if not part of the bucket owner account', - canned: 'private', id: accountToVet, + canned: 'private', + id: accountToVet, authInfo: altAcctAuthInfo, aclParam: null, response: deniedAccess, }, { it: 'should allow bucket owner', - canned: 'private', id: bucketOwnerCanonicalId, + canned: 'private', + id: bucketOwnerCanonicalId, aclParam: null, response: allowedAccess, }, { it: 'should not allow public if canned private', - canned: 'private', id: constants.publicId, + canned: 'private', + id: constants.publicId, aclParam: null, response: deniedAccess, }, { it: 'should not allow other accounts if canned private', - canned: 'private', id: accountToVet, + canned: 'private', + id: accountToVet, aclParam: null, response: deniedAccess, }, { it: 'should allow public if bucket is canned public-read', - canned: 'public-read', id: constants.publicId, + canned: 'public-read', + id: constants.publicId, aclParam: null, response: allowedAccess, }, { it: 'should allow public if bucket is canned public-read-write', - canned: 'public-read-write', id: constants.publicId, + canned: 'public-read-write', + id: constants.publicId, aclParam: null, response: allowedAccess, }, { - it: 'should not allow public if bucket is canned ' + - 'authenticated-read', - canned: 'authenticated-read', id: constants.publicId, + it: 'should not allow public if bucket is canned ' + 'authenticated-read', + canned: 'authenticated-read', + id: constants.publicId, aclParam: null, response: deniedAccess, }, { - it: 'should allow authenticated users if bucket is canned ' + - 'authenticated-read', - canned: 'authenticated-read', id: accountToVet, + it: 'should allow authenticated users if bucket is canned ' + 'authenticated-read', + canned: 'authenticated-read', + id: accountToVet, aclParam: null, response: allowedAccess, }, { it: 'should allow account if granted bucket READ', - canned: '', id: accountToVet, + canned: '', + id: accountToVet, aclParam: ['READ', accountToVet], response: allowedAccess, }, { it: 'should allow account if granted bucket FULL_CONTROL', - canned: '', id: accountToVet, + canned: '', + id: accountToVet, aclParam: ['FULL_CONTROL', accountToVet], response: allowedAccess, }, { it: 'should allow public if granted bucket read action in policy', - canned: 'private', id: constants.publicId, + canned: 'private', + id: constants.publicId, aclParam: null, policy: { Version: '2012-10-17', @@ -388,7 +397,8 @@ describe('without object metadata', () => { }, { it: 'should not allow public if denied bucket read action in policy', - canned: 'public-read', id: constants.publicId, + canned: 'public-read', + id: constants.publicId, aclParam: null, policy: { Version: '2012-10-17', @@ -405,7 +415,8 @@ describe('without object metadata', () => { }, { it: 'should allow account if granted bucket read action in policy', - canned: 'private', id: accountToVet, + canned: 'private', + id: accountToVet, aclParam: null, policy: { Version: '2012-10-17', @@ -423,7 +434,8 @@ describe('without object metadata', () => { }, { it: 'should not allow account if denied bucket read action in policy', - canned: 'public-read', id: accountToVet, + canned: 'public-read', + id: accountToVet, aclParam: null, policy: { Version: '2012-10-17', @@ -454,8 +466,7 @@ describe('without object metadata', () => { } bucket.setCannedAcl(value.canned); - const results = requestTypes.map(type => - isObjAuthorized(bucket, null, type, value.id, authInfoUser, log)); + const results = requestTypes.map(type => isObjAuthorized(bucket, null, type, value.id, authInfoUser, log)); assert.deepStrictEqual(results, value.response); done(); }); @@ -473,8 +484,7 @@ describe('without object metadata', () => { }, ], }); - const results = isObjAuthorized(bucket, null, 'initiateMultipartUpload', - accountToVet, altAcctAuthInfo, log); + const results = isObjAuthorized(bucket, null, 'initiateMultipartUpload', accountToVet, altAcctAuthInfo, log); assert.strictEqual(results, true); }); @@ -490,8 +500,7 @@ describe('without object metadata', () => { }, ], }); - const results = isObjAuthorized(bucket, null, 'objectPutPart', - accountToVet, altAcctAuthInfo, log); + const results = isObjAuthorized(bucket, null, 'objectPutPart', accountToVet, altAcctAuthInfo, log); assert.strictEqual(results, true); }); @@ -507,8 +516,7 @@ describe('without object metadata', () => { }, ], }); - const results = isObjAuthorized(bucket, null, 'completeMultipartUpload', - accountToVet, altAcctAuthInfo, log); + const results = isObjAuthorized(bucket, null, 'completeMultipartUpload', accountToVet, altAcctAuthInfo, log); assert.strictEqual(results, true); }); }); diff --git a/tests/unit/api/objectCopy.js b/tests/unit/api/objectCopy.js index 2448f3276f..55eb6fc099 100644 --- a/tests/unit/api/objectCopy.js +++ b/tests/unit/api/objectCopy.js @@ -9,8 +9,7 @@ const bucketPutPolicy = require('../../../lib/api/bucketPutPolicy'); const objectPut = require('../../../lib/api/objectPut'); const objectCopy = require('../../../lib/api/objectCopy'); const DummyRequest = require('../DummyRequest'); -const { cleanup, DummyRequestLogger, makeAuthInfo, versioningTestUtils } - = require('../helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo, versioningTestUtils } = require('../helpers'); const mpuUtils = require('../utils/mpuUtils'); const metadata = require('../metadataswitch'); const { data } = require('../../../lib/data/wrapper'); @@ -18,9 +17,7 @@ const { objectLocationConstraintHeader } = require('../../../constants'); const { fakeMetadataArchive } = require('../../functional/aws-node-sdk/test/utils/init'); const { config } = require('../../../lib/Config'); -const { - LOCATION_NAME_CRR, -} = require('../../constants'); +const { LOCATION_NAME_CRR } = require('../../constants'); const any = sinon.match.any; @@ -58,50 +55,41 @@ function _createObjectCopyRequest(destBucketName, headers = {}) { const putDestBucketRequest = _createBucketPutRequest(destBucketName); const putSourceBucketRequest = _createBucketPutRequest(sourceBucketName); -const enableVersioningRequest = versioningTestUtils - .createBucketPutVersioningReq(destBucketName, 'Enabled'); -const suspendVersioningRequest = versioningTestUtils - .createBucketPutVersioningReq(destBucketName, 'Suspended'); -const objData = ['foo0', 'foo1', 'foo2'].map(str => - Buffer.from(str, 'utf8')); - +const enableVersioningRequest = versioningTestUtils.createBucketPutVersioningReq(destBucketName, 'Enabled'); +const suspendVersioningRequest = versioningTestUtils.createBucketPutVersioningReq(destBucketName, 'Suspended'); +const objData = ['foo0', 'foo1', 'foo2'].map(str => Buffer.from(str, 'utf8')); describe('objectCopy with versioning', () => { - const testPutObjectRequests = objData.slice(0, 2).map(data => - versioningTestUtils.createPutObjectRequest(destBucketName, objectKey, - data)); - testPutObjectRequests.push(versioningTestUtils - .createPutObjectRequest(sourceBucketName, objectKey, objData[2])); + const testPutObjectRequests = objData + .slice(0, 2) + .map(data => versioningTestUtils.createPutObjectRequest(destBucketName, objectKey, data)); + testPutObjectRequests.push(versioningTestUtils.createPutObjectRequest(sourceBucketName, objectKey, objData[2])); before(done => { cleanup(); sinon.spy(metadata, 'putObjectMD'); - async.series([ - callback => bucketPut(authInfo, putDestBucketRequest, log, - callback), - callback => bucketPut(authInfo, putSourceBucketRequest, log, - callback), - // putting null version: put obj before versioning configured - // in dest bucket - callback => objectPut(authInfo, testPutObjectRequests[0], - undefined, log, callback), - callback => bucketPutVersioning(authInfo, - enableVersioningRequest, log, callback), - // put another version in dest bucket: - callback => objectPut(authInfo, testPutObjectRequests[1], - undefined, log, callback), - callback => bucketPutVersioning(authInfo, - suspendVersioningRequest, log, callback), - // put source object in source bucket - callback => objectPut(authInfo, testPutObjectRequests[2], - undefined, log, callback), - ], err => { - if (err) { - return done(err); - } - versioningTestUtils.assertDataStoreValues(ds, objData); - return done(); - }); + async.series( + [ + callback => bucketPut(authInfo, putDestBucketRequest, log, callback), + callback => bucketPut(authInfo, putSourceBucketRequest, log, callback), + // putting null version: put obj before versioning configured + // in dest bucket + callback => objectPut(authInfo, testPutObjectRequests[0], undefined, log, callback), + callback => bucketPutVersioning(authInfo, enableVersioningRequest, log, callback), + // put another version in dest bucket: + callback => objectPut(authInfo, testPutObjectRequests[1], undefined, log, callback), + callback => bucketPutVersioning(authInfo, suspendVersioningRequest, log, callback), + // put source object in source bucket + callback => objectPut(authInfo, testPutObjectRequests[2], undefined, log, callback), + ], + err => { + if (err) { + return done(err); + } + versioningTestUtils.assertDataStoreValues(ds, objData); + return done(); + }, + ); }); after(() => { @@ -109,13 +97,14 @@ describe('objectCopy with versioning', () => { cleanup(); }); - it('should delete null version when creating new null version, ' + - 'even when null version is not the latest version', done => { - // will have another copy of last object in datastore after objectCopy - const expectedValues = [undefined, objData[1], objData[2], objData[2]]; - const testObjectCopyRequest = _createObjectCopyRequest(destBucketName); - objectCopy(authInfo, testObjectCopyRequest, sourceBucketName, objectKey, - undefined, log, err => { + it( + 'should delete null version when creating new null version, ' + + 'even when null version is not the latest version', + done => { + // will have another copy of last object in datastore after objectCopy + const expectedValues = [undefined, objData[1], objData[2], objData[2]]; + const testObjectCopyRequest = _createObjectCopyRequest(destBucketName); + objectCopy(authInfo, testObjectCopyRequest, sourceBucketName, objectKey, undefined, log, err => { assert.ifError(err, `Unexpected err: ${err}`); sinon.assert.calledWith( metadata.putObjectMD.lastCall, @@ -124,44 +113,42 @@ describe('objectCopy with versioning', () => { sinon.match({ _data: { originOp: 's3:ObjectCreated:Copy' } }), sinon.match.any, sinon.match.any, - sinon.match.any + sinon.match.any, ); setImmediate(() => { - versioningTestUtils - .assertDataStoreValues(ds, expectedValues); + versioningTestUtils.assertDataStoreValues(ds, expectedValues); done(); }); }); - }); + }, + ); it('should not copy object with storage-class header not equal to STANDARD', done => { const testObjectCopyRequest = _createObjectCopyRequest(destBucketName); testObjectCopyRequest.headers['x-amz-storage-class'] = 'COLD'; - objectCopy(authInfo, testObjectCopyRequest, sourceBucketName, objectKey, - undefined, log, err => { - setImmediate(() => { - assert.strictEqual(err.is.InvalidStorageClass, true); - done(); - }); + objectCopy(authInfo, testObjectCopyRequest, sourceBucketName, objectKey, undefined, log, err => { + setImmediate(() => { + assert.strictEqual(err.is.InvalidStorageClass, true); + done(); }); + }); }); it('should not set bucketOwnerId if requesting account owns dest bucket', done => { const testObjectCopyRequest = _createObjectCopyRequest(destBucketName); - objectCopy(authInfo, testObjectCopyRequest, sourceBucketName, objectKey, - undefined, log, err => { - assert.ifError(err); - sinon.assert.calledWith( - metadata.putObjectMD.lastCall, - destBucketName, - objectKey, - sinon.match({ _data: { bucketOwnerId: sinon.match.typeOf('undefined') } }), - sinon.match.any, - sinon.match.any, - sinon.match.any - ); - done(); - }); + objectCopy(authInfo, testObjectCopyRequest, sourceBucketName, objectKey, undefined, log, err => { + assert.ifError(err); + sinon.assert.calledWith( + metadata.putObjectMD.lastCall, + destBucketName, + objectKey, + sinon.match({ _data: { bucketOwnerId: sinon.match.typeOf('undefined') } }), + sinon.match.any, + sinon.match.any, + sinon.match.any, + ); + done(); + }); }); // TODO: S3C-9965 @@ -184,9 +171,7 @@ describe('objectCopy with versioning', () => { Effect: 'Allow', Principal: { AWS: `arn:aws:iam::${authInfo2.shortid}:root` }, Action: ['s3:GetObject'], - Resource: [ - `arn:aws:s3:::${sourceBucketName}/*`, - ], + Resource: [`arn:aws:s3:::${sourceBucketName}/*`], }, ], }), @@ -205,9 +190,7 @@ describe('objectCopy with versioning', () => { Effect: 'Allow', Principal: { AWS: `arn:aws:iam::${authInfo2.shortid}:root` }, Action: ['s3:PutObject'], - Resource: [ - `arn:aws:s3:::${destBucketName}/*`, - ], + Resource: [`arn:aws:s3:::${destBucketName}/*`], }, ], }), @@ -216,51 +199,47 @@ describe('objectCopy with versioning', () => { assert.ifError(err); bucketPutPolicy(authInfo, testPutDestPolicyRequest, log, err => { assert.ifError(err); - objectCopy(authInfo2, testObjectCopyRequest, sourceBucketName, objectKey, - undefined, log, err => { - sinon.assert.calledWith( - metadata.putObjectMD.lastCall, - destBucketName, - objectKey, - sinon.match({ _data: { bucketOwnerId: authInfo.canonicalID } }), - sinon.match.any, - sinon.match.any, - sinon.match.any - ); - assert.ifError(err); - done(); - }); + objectCopy(authInfo2, testObjectCopyRequest, sourceBucketName, objectKey, undefined, log, err => { + sinon.assert.calledWith( + metadata.putObjectMD.lastCall, + destBucketName, + objectKey, + sinon.match({ _data: { bucketOwnerId: authInfo.canonicalID } }), + sinon.match.any, + sinon.match.any, + sinon.match.any, + ); + assert.ifError(err); + done(); + }); }); }); }); }); describe('non-versioned objectCopy', () => { - const testPutObjectRequest = versioningTestUtils - .createPutObjectRequest(sourceBucketName, objectKey, objData[0]); - const testPutDestObjectRequest = versioningTestUtils - .createPutObjectRequest(destBucketName, objectKey, objData[1]); + const testPutObjectRequest = versioningTestUtils.createPutObjectRequest(sourceBucketName, objectKey, objData[0]); + const testPutDestObjectRequest = versioningTestUtils.createPutObjectRequest(destBucketName, objectKey, objData[1]); before(done => { cleanup(); - sinon.stub(metadata, 'putObjectMD') - .callsFake(originalputObjectMD); - - async.series([ - callback => bucketPut(authInfo, putDestBucketRequest, log, - callback), - callback => bucketPut(authInfo, putSourceBucketRequest, log, - callback), - // put source object in source bucket - callback => objectPut(authInfo, testPutObjectRequest, - undefined, log, callback), - ], err => { - if (err) { - return done(err); - } - versioningTestUtils.assertDataStoreValues(ds, objData.slice(0, 1)); - return done(); - }); + sinon.stub(metadata, 'putObjectMD').callsFake(originalputObjectMD); + + async.series( + [ + callback => bucketPut(authInfo, putDestBucketRequest, log, callback), + callback => bucketPut(authInfo, putSourceBucketRequest, log, callback), + // put source object in source bucket + callback => objectPut(authInfo, testPutObjectRequest, undefined, log, callback), + ], + err => { + if (err) { + return done(err); + } + versioningTestUtils.assertDataStoreValues(ds, objData.slice(0, 1)); + return done(); + }, + ); }); after(() => { @@ -271,91 +250,132 @@ describe('non-versioned objectCopy', () => { const testObjectCopyRequest = _createObjectCopyRequest(destBucketName); it('should not leave orphans in data when overwriting a multipart upload', done => { - mpuUtils.createMPU(namespace, destBucketName, objectKey, log, - (err, testUploadId) => { + mpuUtils.createMPU(namespace, destBucketName, objectKey, log, (err, testUploadId) => { assert.ifError(err); - objectCopy(authInfo, testObjectCopyRequest, sourceBucketName, objectKey, - undefined, log, err => { - assert.ifError(err); - sinon.assert.calledWith(metadata.putObjectMD, - any, any, any, sinon.match({ oldReplayId: testUploadId }), any, any); - done(); - }); + objectCopy(authInfo, testObjectCopyRequest, sourceBucketName, objectKey, undefined, log, err => { + assert.ifError(err); + sinon.assert.calledWith( + metadata.putObjectMD, + any, + any, + any, + sinon.match({ oldReplayId: testUploadId }), + any, + any, + ); + done(); + }); }); }); it('should not pass needOplogUpdate when creating object', done => { - async.series([ - next => objectCopy(authInfo, testObjectCopyRequest, sourceBucketName, objectKey, - undefined, log, next), - async () => { - sinon.assert.calledWith(metadata.putObjectMD.lastCall, - destBucketName, objectKey, sinon.match({ - _data: { originOp: 's3:ObjectCreated:Copy' }, - }), sinon.match({ - needOplogUpdate: undefined, - originOp: undefined, - }), any, any); - }, - ], done); + async.series( + [ + next => objectCopy(authInfo, testObjectCopyRequest, sourceBucketName, objectKey, undefined, log, next), + async () => { + sinon.assert.calledWith( + metadata.putObjectMD.lastCall, + destBucketName, + objectKey, + sinon.match({ + _data: { originOp: 's3:ObjectCreated:Copy' }, + }), + sinon.match({ + needOplogUpdate: undefined, + originOp: undefined, + }), + any, + any, + ); + }, + ], + done, + ); }); it('should not pass needOplogUpdate when replacing object', done => { - async.series([ - next => objectPut(authInfo, testPutDestObjectRequest, undefined, log, next), - next => objectCopy(authInfo, testObjectCopyRequest, sourceBucketName, objectKey, - undefined, log, next), - async () => { - sinon.assert.calledWith(metadata.putObjectMD.lastCall, - destBucketName, objectKey, sinon.match({ - _data: { originOp: 's3:ObjectCreated:Copy' }, - }), sinon.match({ - needOplogUpdate: undefined, - originOp: undefined, - }), any, any); - }, - ], done); + async.series( + [ + next => objectPut(authInfo, testPutDestObjectRequest, undefined, log, next), + next => objectCopy(authInfo, testObjectCopyRequest, sourceBucketName, objectKey, undefined, log, next), + async () => { + sinon.assert.calledWith( + metadata.putObjectMD.lastCall, + destBucketName, + objectKey, + sinon.match({ + _data: { originOp: 's3:ObjectCreated:Copy' }, + }), + sinon.match({ + needOplogUpdate: undefined, + originOp: undefined, + }), + any, + any, + ); + }, + ], + done, + ); }); it('should pass needOplogUpdate to metadata when replacing archived object', done => { const archived = { - archiveInfo: { foo: 0, bar: 'stuff' } + archiveInfo: { foo: 0, bar: 'stuff' }, }; - async.series([ - next => objectPut(authInfo, testPutDestObjectRequest, undefined, log, next), - next => fakeMetadataArchive(destBucketName, objectKey, undefined, archived, next), - next => objectCopy(authInfo, testObjectCopyRequest, sourceBucketName, objectKey, - undefined, log, next), - async () => { - sinon.assert.calledWith(metadata.putObjectMD.lastCall, - destBucketName, objectKey, any, sinon.match({ - needOplogUpdate: true, - originOp: 's3:ReplaceArchivedObject', - }), any, any); - }, - ], done); + async.series( + [ + next => objectPut(authInfo, testPutDestObjectRequest, undefined, log, next), + next => fakeMetadataArchive(destBucketName, objectKey, undefined, archived, next), + next => objectCopy(authInfo, testObjectCopyRequest, sourceBucketName, objectKey, undefined, log, next), + async () => { + sinon.assert.calledWith( + metadata.putObjectMD.lastCall, + destBucketName, + objectKey, + any, + sinon.match({ + needOplogUpdate: true, + originOp: 's3:ReplaceArchivedObject', + }), + any, + any, + ); + }, + ], + done, + ); }); it('should pass needOplogUpdate to metadata when replacing archived object in version suspended bucket', done => { const archived = { - archiveInfo: { foo: 0, bar: 'stuff' } + archiveInfo: { foo: 0, bar: 'stuff' }, }; - async.series([ - next => bucketPutVersioning(authInfo, suspendVersioningRequest, log, next), - next => objectPut(authInfo, testPutDestObjectRequest, undefined, log, next), - next => fakeMetadataArchive(destBucketName, objectKey, undefined, archived, next), - next => objectCopy(authInfo, testObjectCopyRequest, sourceBucketName, objectKey, - undefined, log, next), - async () => { - sinon.assert.calledWith(metadata.putObjectMD.lastCall, - destBucketName, objectKey, any, sinon.match({ - needOplogUpdate: true, - originOp: 's3:ReplaceArchivedObject', - }), any, any); - }, - ], done); + async.series( + [ + next => bucketPutVersioning(authInfo, suspendVersioningRequest, log, next), + next => objectPut(authInfo, testPutDestObjectRequest, undefined, log, next), + next => fakeMetadataArchive(destBucketName, objectKey, undefined, archived, next), + next => objectCopy(authInfo, testObjectCopyRequest, sourceBucketName, objectKey, undefined, log, next), + async () => { + sinon.assert.calledWith( + metadata.putObjectMD.lastCall, + destBucketName, + objectKey, + any, + sinon.match({ + needOplogUpdate: true, + originOp: 's3:ReplaceArchivedObject', + }), + any, + any, + ); + }, + ], + done, + ); }); it('should fail to copy object when setting a crr location as the locationConstraint', done => { @@ -364,14 +384,16 @@ describe('non-versioned objectCopy', () => { [objectLocationConstraintHeader]: LOCATION_NAME_CRR, }); - async.series([ - next => objectPut(authInfo, testPutDestObjectRequest, undefined, log, next), - next => objectCopy(authInfo, testObjectCopyRequest, sourceBucketName, objectKey, - undefined, log, next), - ], err => { - assert(err.is.InvalidArgument); - done(); - }); + async.series( + [ + next => objectPut(authInfo, testPutDestObjectRequest, undefined, log, next), + next => objectCopy(authInfo, testObjectCopyRequest, sourceBucketName, objectKey, undefined, log, next), + ], + err => { + assert(err.is.InvalidArgument); + done(); + }, + ); }); }); @@ -379,10 +401,13 @@ describe('objectCopy overheadField', () => { beforeEach(done => { cleanup(); sinon.stub(metadata, 'putObjectMD').callsFake(originalputObjectMD); - async.series([ - next => bucketPut(authInfo, putSourceBucketRequest, log, next), - next => bucketPut(authInfo, putDestBucketRequest, log, next), - ], done); + async.series( + [ + next => bucketPut(authInfo, putSourceBucketRequest, log, next), + next => bucketPut(authInfo, putDestBucketRequest, log, next), + ], + done, + ); }); afterEach(() => { @@ -391,62 +416,82 @@ describe('objectCopy overheadField', () => { }); it('should pass overheadField to metadata.putObjectMD for a non-versioned request', done => { - const testPutObjectRequest = - versioningTestUtils.createPutObjectRequest(sourceBucketName, objectKey, objData[0]); + const testPutObjectRequest = versioningTestUtils.createPutObjectRequest( + sourceBucketName, + objectKey, + objData[0], + ); const testObjectCopyRequest = _createObjectCopyRequest(destBucketName); objectPut(authInfo, testPutObjectRequest, undefined, log, err => { assert.ifError(err); - objectCopy(authInfo, testObjectCopyRequest, sourceBucketName, objectKey, undefined, log, - err => { - assert.ifError(err); - sinon.assert.calledWith(metadata.putObjectMD.lastCall, - destBucketName, objectKey, any, sinon.match({ overheadField: sinon.match.array }), any, any); - done(); - } - ); + objectCopy(authInfo, testObjectCopyRequest, sourceBucketName, objectKey, undefined, log, err => { + assert.ifError(err); + sinon.assert.calledWith( + metadata.putObjectMD.lastCall, + destBucketName, + objectKey, + any, + sinon.match({ overheadField: sinon.match.array }), + any, + any, + ); + done(); + }); }); }); it('should pass overheadField to metadata.putObjectMD for a versioned request', done => { - const testPutObjectRequest = - versioningTestUtils.createPutObjectRequest(sourceBucketName, objectKey, objData[0]); + const testPutObjectRequest = versioningTestUtils.createPutObjectRequest( + sourceBucketName, + objectKey, + objData[0], + ); const testObjectCopyRequest = _createObjectCopyRequest(destBucketName); objectPut(authInfo, testPutObjectRequest, undefined, log, err => { assert.ifError(err); bucketPutVersioning(authInfo, enableVersioningRequest, log, err => { assert.ifError(err); - objectCopy(authInfo, testObjectCopyRequest, sourceBucketName, objectKey, undefined, log, - err => { - assert.ifError(err); - sinon.assert.calledWith(metadata.putObjectMD.lastCall, - destBucketName, objectKey, any, - sinon.match({ overheadField: sinon.match.array }), any, any - ); - done(); - } - ); + objectCopy(authInfo, testObjectCopyRequest, sourceBucketName, objectKey, undefined, log, err => { + assert.ifError(err); + sinon.assert.calledWith( + metadata.putObjectMD.lastCall, + destBucketName, + objectKey, + any, + sinon.match({ overheadField: sinon.match.array }), + any, + any, + ); + done(); + }); }); }); }); it('should pass overheadField to metadata.putObjectMD for a version-suspended request', done => { - const testPutObjectRequest = - versioningTestUtils.createPutObjectRequest(sourceBucketName, objectKey, objData[0]); + const testPutObjectRequest = versioningTestUtils.createPutObjectRequest( + sourceBucketName, + objectKey, + objData[0], + ); const testObjectCopyRequest = _createObjectCopyRequest(destBucketName); objectPut(authInfo, testPutObjectRequest, undefined, log, err => { assert.ifError(err); bucketPutVersioning(authInfo, suspendVersioningRequest, log, err => { assert.ifError(err); - objectCopy(authInfo, testObjectCopyRequest, sourceBucketName, objectKey, undefined, log, - err => { - assert.ifError(err); - sinon.assert.calledWith(metadata.putObjectMD.lastCall, - destBucketName, objectKey, any, - sinon.match({ overheadField: sinon.match.array }), any, any - ); - done(); - } - ); + objectCopy(authInfo, testObjectCopyRequest, sourceBucketName, objectKey, undefined, log, err => { + assert.ifError(err); + sinon.assert.calledWith( + metadata.putObjectMD.lastCall, + destBucketName, + objectKey, + any, + sinon.match({ overheadField: sinon.match.array }), + any, + any, + ); + done(); + }); }); }); }); @@ -461,10 +506,16 @@ describe('objectCopy in ingestion bucket', () => { before(() => { // Setup multi-backend, this is required for ingestion - data.switch(new storage.data.MultipleBackendGateway({ - 'us-east-1': dataClient, - 'us-east-2': dataClient, - }, metadata, data.locStorageCheckFn)); + data.switch( + new storage.data.MultipleBackendGateway( + { + 'us-east-1': dataClient, + 'us-east-2': dataClient, + }, + metadata, + data.locStorageCheckFn, + ), + ); data.implName = 'multipleBackends'; // "mock" the data location, simulating a backend supporting server-side copy @@ -497,19 +548,20 @@ describe('objectCopy in ingestion bucket', () => { sinon.restore(); }); - const newPutIngestBucketRequest = location => new DummyRequest({ - bucketName: destBucketName, - namespace, - headers: { host: `${destBucketName}.s3.amazonaws.com` }, - url: '/', - post: '' + - '' + - `${location}` + - '', - }); - const putSourceObjectRequest = versioningTestUtils.createPutObjectRequest( - sourceBucketName, objectKey, objData[0]); + const newPutIngestBucketRequest = location => + new DummyRequest({ + bucketName: destBucketName, + namespace, + headers: { host: `${destBucketName}.s3.amazonaws.com` }, + url: '/', + post: + '' + + '' + + `${location}` + + '', + }); + const putSourceObjectRequest = versioningTestUtils.createPutObjectRequest(sourceBucketName, objectKey, objData[0]); const newPutObjectRequest = params => { const { location } = params || {}; const r = _createObjectCopyRequest(destBucketName); @@ -526,17 +578,28 @@ describe('objectCopy in ingestion bucket', () => { const versionID = versioning.VersionID.encode(versioning.VersionID.generateVersionId('0', '')); dataClient.copyObject = sinon.stub().yields(null, objectKey, versionID); - async.series([ - next => bucketPut(authInfo, putSourceBucketRequest, log, next), - next => bucketPut(authInfo, newPutIngestBucketRequest('us-east-1:ingest'), log, next), - next => objectPut(authInfo, putSourceObjectRequest, undefined, log, next), - next => objectCopy(authInfo, newPutObjectRequest(), sourceBucketName, objectKey, undefined, log, - (err, xml, headers) => { - assert.ifError(err); - assert.strictEqual(headers['x-amz-version-id'], versionID); - next(); - }), - ], done); + async.series( + [ + next => bucketPut(authInfo, putSourceBucketRequest, log, next), + next => bucketPut(authInfo, newPutIngestBucketRequest('us-east-1:ingest'), log, next), + next => objectPut(authInfo, putSourceObjectRequest, undefined, log, next), + next => + objectCopy( + authInfo, + newPutObjectRequest(), + sourceBucketName, + objectKey, + undefined, + log, + (err, xml, headers) => { + assert.ifError(err); + assert.strictEqual(headers['x-amz-version-id'], versionID); + next(); + }, + ), + ], + done, + ); }); it('should not use the versionID from the backend when writing in another location', done => { @@ -544,34 +607,56 @@ describe('objectCopy in ingestion bucket', () => { dataClient.copyObject = sinon.stub().yields(null, objectKey, versionID); const copyObjectRequest = newPutObjectRequest({ location: 'us-east-2' }); - async.series([ - next => bucketPut(authInfo, putSourceBucketRequest, log, next), - next => bucketPut(authInfo, newPutIngestBucketRequest('us-east-1:ingest'), log, next), - next => objectPut(authInfo, putSourceObjectRequest, undefined, log, next), - next => objectCopy(authInfo, copyObjectRequest, sourceBucketName, objectKey, undefined, log, - (err, xml, headers) => { - assert.ifError(err); - assert.notEqual(headers['x-amz-version-id'], versionID); - next(); - }), - ], done); + async.series( + [ + next => bucketPut(authInfo, putSourceBucketRequest, log, next), + next => bucketPut(authInfo, newPutIngestBucketRequest('us-east-1:ingest'), log, next), + next => objectPut(authInfo, putSourceObjectRequest, undefined, log, next), + next => + objectCopy( + authInfo, + copyObjectRequest, + sourceBucketName, + objectKey, + undefined, + log, + (err, xml, headers) => { + assert.ifError(err); + assert.notEqual(headers['x-amz-version-id'], versionID); + next(); + }, + ), + ], + done, + ); }); it('should not use the versionID from the backend when it is not a valid versionID', done => { const versionID = undefined; dataClient.copyObject = sinon.stub().yields(null, objectKey, versionID); - async.series([ - next => bucketPut(authInfo, putSourceBucketRequest, log, next), - next => bucketPut(authInfo, newPutIngestBucketRequest('us-east-1:ingest'), log, next), - next => objectPut(authInfo, putSourceObjectRequest, undefined, log, next), - next => objectCopy(authInfo, newPutObjectRequest(), sourceBucketName, objectKey, undefined, log, - (err, xml, headers) => { - assert.ifError(err); - assert.notEqual(headers['x-amz-version-id'], versionID); - next(); - }), - ], done); + async.series( + [ + next => bucketPut(authInfo, putSourceBucketRequest, log, next), + next => bucketPut(authInfo, newPutIngestBucketRequest('us-east-1:ingest'), log, next), + next => objectPut(authInfo, putSourceObjectRequest, undefined, log, next), + next => + objectCopy( + authInfo, + newPutObjectRequest(), + sourceBucketName, + objectKey, + undefined, + log, + (err, xml, headers) => { + assert.ifError(err); + assert.notEqual(headers['x-amz-version-id'], versionID); + next(); + }, + ), + ], + done, + ); }); }); @@ -580,12 +665,21 @@ describe('objectCopy with objectKeyByteLimit', () => { beforeEach(done => { cleanup(); - async.series([ - next => bucketPut(authInfo, putDestBucketRequest, log, next), - next => bucketPut(authInfo, putSourceBucketRequest, log, next), - next => objectPut(authInfo, versioningTestUtils.createPutObjectRequest( - sourceBucketName, objectKey, objData[0]), undefined, log, next), - ], done); + async.series( + [ + next => bucketPut(authInfo, putDestBucketRequest, log, next), + next => bucketPut(authInfo, putSourceBucketRequest, log, next), + next => + objectPut( + authInfo, + versioningTestUtils.createPutObjectRequest(sourceBucketName, objectKey, objData[0]), + undefined, + log, + next, + ), + ], + done, + ); }); afterEach(() => { @@ -598,13 +692,12 @@ describe('objectCopy with objectKeyByteLimit', () => { testCopyObjectRequest.objectKey = longDestKey; testCopyObjectRequest.url = `/${destBucketName}/${longDestKey}`; - objectCopy(authInfo, testCopyObjectRequest, sourceBucketName, objectKey, - undefined, log, err => { - assert(err); - assert.strictEqual(err.KeyTooLong, true); - assert.match(err.description, /915/); - done(); - }); + objectCopy(authInfo, testCopyObjectRequest, sourceBucketName, objectKey, undefined, log, err => { + assert(err); + assert.strictEqual(err.KeyTooLong, true); + assert.match(err.description, /915/); + done(); + }); }); it('should accept destination object key longer than 915 bytes with objectKeyByteLimit', done => { @@ -615,12 +708,11 @@ describe('objectCopy with objectKeyByteLimit', () => { testCopyObjectRequest.objectKey = longDestKey; testCopyObjectRequest.url = `/${destBucketName}/${longDestKey}`; - objectCopy(authInfo, testCopyObjectRequest, sourceBucketName, objectKey, - undefined, log, (err, xml) => { - assert.ifError(err); - assert(xml); - done(); - }); + objectCopy(authInfo, testCopyObjectRequest, sourceBucketName, objectKey, undefined, log, (err, xml) => { + assert.ifError(err); + assert(xml); + done(); + }); }); it('should reject destination object key exceeding objectKeyByteLimit', done => { @@ -631,12 +723,11 @@ describe('objectCopy with objectKeyByteLimit', () => { testCopyObjectRequest.objectKey = longDestKey; testCopyObjectRequest.url = `/${destBucketName}/${longDestKey}`; - objectCopy(authInfo, testCopyObjectRequest, sourceBucketName, objectKey, - undefined, log, err => { - assert(err); - assert.strictEqual(err.KeyTooLong, true); - assert.match(err.description, /1024/); - done(); - }); + objectCopy(authInfo, testCopyObjectRequest, sourceBucketName, objectKey, undefined, log, err => { + assert(err); + assert.strictEqual(err.KeyTooLong, true); + assert.match(err.description, /1024/); + done(); + }); }); }); diff --git a/tests/unit/api/objectCopyPart.js b/tests/unit/api/objectCopyPart.js index 46c95a0452..bf6a6cc19a 100644 --- a/tests/unit/api/objectCopyPart.js +++ b/tests/unit/api/objectCopyPart.js @@ -6,13 +6,11 @@ const { storage } = require('arsenal'); const { bucketPut } = require('../../../lib/api/bucketPut'); const objectPut = require('../../../lib/api/objectPut'); const objectPutCopyPart = require('../../../lib/api/objectPutCopyPart'); -const initiateMultipartUpload -= require('../../../lib/api/initiateMultipartUpload'); +const initiateMultipartUpload = require('../../../lib/api/initiateMultipartUpload'); const { metadata } = storage.metadata.inMemory.metadata; const metadataswitch = require('../metadataswitch'); const DummyRequest = require('../DummyRequest'); -const { cleanup, DummyRequestLogger, makeAuthInfo, versioningTestUtils } - = require('../helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo, versioningTestUtils } = require('../helpers'); const log = new DummyRequestLogger(); const canonicalID = 'accessKey1'; @@ -64,30 +62,27 @@ const initiateRequest = _createInitiateRequest(destBucketName); describe('objectCopyPart', () => { let uploadId; const objData = Buffer.from('foo', 'utf8'); - const testPutObjectRequest = - versioningTestUtils.createPutObjectRequest(sourceBucketName, objectKey, - objData); + const testPutObjectRequest = versioningTestUtils.createPutObjectRequest(sourceBucketName, objectKey, objData); before(done => { cleanup(); sinon.spy(metadataswitch, 'putObjectMD'); - async.waterfall([ - callback => bucketPut(authInfo, putDestBucketRequest, log, - err => callback(err)), - callback => bucketPut(authInfo, putSourceBucketRequest, log, - err => callback(err)), - callback => objectPut(authInfo, testPutObjectRequest, - undefined, log, err => callback(err)), - callback => initiateMultipartUpload(authInfo, initiateRequest, - log, (err, res) => callback(err, res)), - ], (err, res) => { - if (err) { - return done(err); - } - return parseString(res, (err, json) => { - uploadId = json.InitiateMultipartUploadResult.UploadId[0]; - return done(); - }); - }); + async.waterfall( + [ + callback => bucketPut(authInfo, putDestBucketRequest, log, err => callback(err)), + callback => bucketPut(authInfo, putSourceBucketRequest, log, err => callback(err)), + callback => objectPut(authInfo, testPutObjectRequest, undefined, log, err => callback(err)), + callback => initiateMultipartUpload(authInfo, initiateRequest, log, (err, res) => callback(err, res)), + ], + (err, res) => { + if (err) { + return done(err); + } + return parseString(res, (err, json) => { + uploadId = json.InitiateMultipartUploadResult.UploadId[0]; + return done(); + }); + }, + ); }); after(() => { @@ -95,8 +90,7 @@ describe('objectCopyPart', () => { cleanup(); }); - it('should copy part even if legacy metadata without dataStoreName', - done => { + it('should copy part even if legacy metadata without dataStoreName', done => { // force metadata for dataStoreName to be undefined metadata.keyMaps.get(sourceBucketName).get(objectKey).dataStoreName = undefined; const testObjectCopyRequest = _createObjectCopyPartRequest(destBucketName, uploadId); @@ -108,17 +102,17 @@ describe('objectCopyPart', () => { it('should return InvalidArgument error given invalid range', done => { const headers = { 'x-amz-copy-source-range': 'bad-range-parameter' }; - const req = - _createObjectCopyPartRequest(destBucketName, uploadId, headers); - objectPutCopyPart( - authInfo, req, sourceBucketName, objectKey, undefined, log, err => { - assert(err.is.InvalidArgument); - assert.strictEqual(err.description, - 'The x-amz-copy-source-range value must be of the form ' + + const req = _createObjectCopyPartRequest(destBucketName, uploadId, headers); + objectPutCopyPart(authInfo, req, sourceBucketName, objectKey, undefined, log, err => { + assert(err.is.InvalidArgument); + assert.strictEqual( + err.description, + 'The x-amz-copy-source-range value must be of the form ' + 'bytes=first-last where first and last are the ' + - 'zero-based offsets of the first and last bytes to copy'); - done(); - }); + 'zero-based offsets of the first and last bytes to copy', + ); + done(); + }); }); it('should pass overheadField', done => { @@ -132,7 +126,7 @@ describe('objectCopyPart', () => { sinon.match.any, sinon.match({ overheadField: sinon.match.array }), sinon.match.any, - sinon.match.any + sinon.match.any, ); done(); }); @@ -149,7 +143,7 @@ describe('objectCopyPart', () => { sinon.match({ 'owner-id': authInfo.canonicalID }), sinon.match.any, sinon.match.any, - sinon.match.any + sinon.match.any, ); done(); }); diff --git a/tests/unit/api/objectDelete.js b/tests/unit/api/objectDelete.js index 2e02455602..5acf3d8fd3 100644 --- a/tests/unit/api/objectDelete.js +++ b/tests/unit/api/objectDelete.js @@ -7,7 +7,7 @@ const services = require('../../../lib/services'); const { bucketPut } = require('../../../lib/api/bucketPut'); const bucketPutACL = require('../../../lib/api/bucketPutACL'); const constants = require('../../../constants'); -const { cleanup, DummyRequestLogger, makeAuthInfo, versioningTestUtils} = require('../helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo, versioningTestUtils } = require('../helpers'); const objectPut = require('../../../lib/api/objectPut'); const { objectDelete, objectDeleteInternal } = require('../../../lib/api/objectDelete'); const objectGet = require('../../../lib/api/objectGet'); @@ -35,8 +35,7 @@ lateDate.setMinutes(lateDate.getMinutes() + 30); const enableVersioningRequest = versioningTestUtils.createBucketPutVersioningReq(bucketName, 'Enabled'); -function testAuth(bucketOwner, authUser, bucketPutReq, objPutReq, objDelReq, - log, cb) { +function testAuth(bucketOwner, authUser, bucketPutReq, objPutReq, objDelReq, log, cb) { bucketPut(bucketOwner, bucketPutReq, log, () => { bucketPutACL(bucketOwner, bucketPutReq, log, err => { assert.strictEqual(err, undefined); @@ -56,13 +55,16 @@ describe('objectDelete API', () => { beforeEach(() => { cleanup(); - testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey, - headers: {}, - url: `/${bucketName}/${objectKey}`, - }, postBody); + testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey, + headers: {}, + url: `/${bucketName}/${objectKey}`, + }, + postBody, + ); sinon.stub(services, 'deleteObject').callsFake(originalDeleteObject); sinon.spy(metadataswitch, 'putObjectMD'); @@ -73,7 +75,6 @@ describe('objectDelete API', () => { sinon.restore(); }); - const testBucketPutRequest = new DummyRequest({ bucketName, namespace, @@ -96,43 +97,65 @@ describe('objectDelete API', () => { }); it('should delete an object', done => { - async.series([ - next => bucketPut(authInfo, testBucketPutRequest, log, next), - next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), - next => objectDelete(authInfo, testDeleteRequest, log, next), - async () => sinon.assert.calledWith(services.deleteObject, - any, any, any, - sinon.match({ - deleteData: true, - doesNotNeedOpogUpdate: true, - }), - any, any, any), - next => objectGet(authInfo, testGetObjectRequest, false, log, err => { - assert.strictEqual(err.is.NoSuchKey, true); - next(); - }), - ], done); + async.series( + [ + next => bucketPut(authInfo, testBucketPutRequest, log, next), + next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), + next => objectDelete(authInfo, testDeleteRequest, log, next), + async () => + sinon.assert.calledWith( + services.deleteObject, + any, + any, + any, + sinon.match({ + deleteData: true, + doesNotNeedOpogUpdate: true, + }), + any, + any, + any, + ), + next => + objectGet(authInfo, testGetObjectRequest, false, log, err => { + assert.strictEqual(err.is.NoSuchKey, true); + next(); + }), + ], + done, + ); }); it('should delete an object with oplog update when object is archived', done => { const archived = { archiveInfo: { foo: 0, bar: 'stuff' } }; - async.series([ - next => bucketPut(authInfo, testBucketPutRequest, log, next), - next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), - next => fakeMetadataArchive(bucketName, objectKey, undefined, archived, next), - next => objectDelete(authInfo, testDeleteRequest, log, next), - async () => sinon.assert.calledWith(services.deleteObject, - any, any, any, - sinon.match({ - deleteData: true, - doesNotNeedOpogUpdate: undefined, - }), - any, any, any), - next => objectGet(authInfo, testGetObjectRequest, false, log, err => { - assert.strictEqual(err.is.NoSuchKey, true); - next(); - }), - ], done); + async.series( + [ + next => bucketPut(authInfo, testBucketPutRequest, log, next), + next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), + next => fakeMetadataArchive(bucketName, objectKey, undefined, archived, next), + next => objectDelete(authInfo, testDeleteRequest, log, next), + async () => + sinon.assert.calledWith( + services.deleteObject, + any, + any, + any, + sinon.match({ + deleteData: true, + doesNotNeedOpogUpdate: undefined, + }), + any, + any, + any, + ), + next => + objectGet(authInfo, testGetObjectRequest, false, log, err => { + assert.strictEqual(err.is.NoSuchKey, true); + next(); + }), + ], + done, + ); }); it('should delete an object with oplog update when bucket has bucket notification', done => { @@ -141,72 +164,90 @@ describe('objectDelete API', () => { headers: { host: `${bucketName}.s3.amazonaws.com`, }, - post: '' + + post: + '' + '', actionImplicitDenies: false, }; - async.series([ - next => bucketPut(authInfo, testBucketPutRequest, log, next), - next => bucketPutNotification(authInfo, putNotifConfigRequest, log, next), - next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), - next => objectDelete(authInfo, testDeleteRequest, log, next), - async () => sinon.assert.calledWith(services.deleteObject, - any, any, any, - sinon.match({ - deleteData: true, - doesNotNeedOpogUpdate: undefined, - }), - any, any, any), - next => objectGet(authInfo, testGetObjectRequest, false, log, err => { - assert.strictEqual(err.is.NoSuchKey, true); - next(); - }), - ], done); + async.series( + [ + next => bucketPut(authInfo, testBucketPutRequest, log, next), + next => bucketPutNotification(authInfo, putNotifConfigRequest, log, next), + next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), + next => objectDelete(authInfo, testDeleteRequest, log, next), + async () => + sinon.assert.calledWith( + services.deleteObject, + any, + any, + any, + sinon.match({ + deleteData: true, + doesNotNeedOpogUpdate: undefined, + }), + any, + any, + any, + ), + next => + objectGet(authInfo, testGetObjectRequest, false, log, err => { + assert.strictEqual(err.is.NoSuchKey, true); + next(); + }), + ], + done, + ); }); it('should delete a 0 bytes object', done => { - const testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey, - headers: {}, - url: `/${bucketName}/${objectKey}`, - }, ''); + const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey, + headers: {}, + url: `/${bucketName}/${objectKey}`, + }, + '', + ); bucketPut(authInfo, testBucketPutRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, - undefined, log, () => { - objectDelete(authInfo, testDeleteRequest, log, err => { - assert.strictEqual(err, null); - objectGet(authInfo, testGetObjectRequest, false, - log, err => { - const expected = - Object.assign({}, errors.NoSuchKey); - const received = Object.assign({}, err); - assert.deepStrictEqual(received, expected); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, () => { + objectDelete(authInfo, testDeleteRequest, log, err => { + assert.strictEqual(err, null); + objectGet(authInfo, testGetObjectRequest, false, log, err => { + const expected = Object.assign({}, errors.NoSuchKey); + const received = Object.assign({}, err); + assert.deepStrictEqual(received, expected); + done(); }); }); + }); }); }); it('should delete a multipart upload and send `uploadId` as `replayId` to deleteObject', done => { bucketPut(authInfo, testBucketPutRequest, log, () => { - mpuUtils.createMPU(namespace, bucketName, objectKey, log, - (err, testUploadId) => { - assert.ifError(err); - objectDelete(authInfo, testDeleteRequest, log, err => { - assert.strictEqual(err, null); - sinon.assert.calledWith(services.deleteObject, - any, any, any, - sinon.match({ - deleteData: true, - replayId: testUploadId, - doesNotNeedOpogUpdate: true, - }), any, any, any); - done(); - }); + mpuUtils.createMPU(namespace, bucketName, objectKey, log, (err, testUploadId) => { + assert.ifError(err); + objectDelete(authInfo, testDeleteRequest, log, err => { + assert.strictEqual(err, null); + sinon.assert.calledWith( + services.deleteObject, + any, + any, + any, + sinon.match({ + deleteData: true, + replayId: testUploadId, + doesNotNeedOpogUpdate: true, + }), + any, + any, + any, + ); + done(); }); + }); }); }); @@ -223,46 +264,40 @@ describe('objectDelete API', () => { it('should del object if user has FULL_CONTROL grant on bucket', done => { const bucketOwner = makeAuthInfo('accessKey2'); const authUser = makeAuthInfo('accessKey3'); - testBucketPutRequest.headers['x-amz-grant-full-control'] = - `id=${authUser.getCanonicalID()}`; - testAuth(bucketOwner, authUser, testBucketPutRequest, - testPutObjectRequest, testDeleteRequest, log, done); + testBucketPutRequest.headers['x-amz-grant-full-control'] = `id=${authUser.getCanonicalID()}`; + testAuth(bucketOwner, authUser, testBucketPutRequest, testPutObjectRequest, testDeleteRequest, log, done); }); it('should del object if user has WRITE grant on bucket', done => { const bucketOwner = makeAuthInfo('accessKey2'); const authUser = makeAuthInfo('accessKey3'); - testBucketPutRequest.headers['x-amz-grant-write'] = - `id=${authUser.getCanonicalID()}`; - testAuth(bucketOwner, authUser, testBucketPutRequest, - testPutObjectRequest, testDeleteRequest, log, done); + testBucketPutRequest.headers['x-amz-grant-write'] = `id=${authUser.getCanonicalID()}`; + testAuth(bucketOwner, authUser, testBucketPutRequest, testPutObjectRequest, testDeleteRequest, log, done); }); it('should del object in bucket with public-read-write acl', done => { const bucketOwner = makeAuthInfo('accessKey2'); const authUser = makeAuthInfo('accessKey3'); testBucketPutRequest.headers['x-amz-acl'] = 'public-read-write'; - testAuth(bucketOwner, authUser, testBucketPutRequest, - testPutObjectRequest, testDeleteRequest, log, done); + testAuth(bucketOwner, authUser, testBucketPutRequest, testPutObjectRequest, testDeleteRequest, log, done); }); it('should pass overheadField to metadata', done => { bucketPut(authInfo, testBucketPutRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, - undefined, log, () => { - objectDelete(authInfo, testDeleteRequest, log, err => { - assert.strictEqual(err, null); - sinon.assert.calledWith( - metadataswitch.deleteObjectMD, - bucketName, - objectKey, - sinon.match({ overheadField: sinon.match.array }), - sinon.match.any, - sinon.match.any - ); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, () => { + objectDelete(authInfo, testDeleteRequest, log, err => { + assert.strictEqual(err, null); + sinon.assert.calledWith( + metadataswitch.deleteObjectMD, + bucketName, + objectKey, + sinon.match({ overheadField: sinon.match.array }), + sinon.match.any, + sinon.match.any, + ); + done(); }); + }); }); }); @@ -276,52 +311,54 @@ describe('objectDelete API', () => { }); bucketPut(authInfo, testBucketPutVersionRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, - undefined, log, (err, data) => { - const deleteObjectVersionRequest = new DummyRequest({ - bucketName, - namespace, - objectKey, - headers: {}, - url: `/${bucketName}/${objectKey}?versionId=${data['x-amz-version-id']}`, - query: { - versionId: data['x-amz-version-id'], - }, - }); - objectDeleteInternal(authInfo, deleteObjectVersionRequest, log, true, err => { - assert.strictEqual(err, null); - sinon.assert.calledWith(warnStub, 'expiration is trying to delete a master version ' + - 'of an object with versioning enabled'); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, data) => { + const deleteObjectVersionRequest = new DummyRequest({ + bucketName, + namespace, + objectKey, + headers: {}, + url: `/${bucketName}/${objectKey}?versionId=${data['x-amz-version-id']}`, + query: { + versionId: data['x-amz-version-id'], + }, }); + objectDeleteInternal(authInfo, deleteObjectVersionRequest, log, true, err => { + assert.strictEqual(err, null); + sinon.assert.calledWith( + warnStub, + 'expiration is trying to delete a master version ' + 'of an object with versioning enabled', + ); + done(); + }); + }); }); }); - describe('with \'modified\' headers', () => { + describe("with 'modified' headers", () => { beforeEach(done => { bucketPut(authInfo, testBucketPutRequest, log, () => { objectPut(authInfo, testPutObjectRequest, undefined, log, done); }); }); - it('should return error if request includes \'if-unmodified-since\' ' + - 'header and object has been modified', done => { - const testDeleteRequest = new DummyRequest({ - bucketName, - namespace, - objectKey, - headers: { 'if-unmodified-since': earlyDate }, - url: `/${bucketName}/${objectKey}`, - }); - objectDelete(authInfo, testDeleteRequest, log, err => { - assert.strictEqual(err.is.PreconditionFailed, true); - done(); - }); - }); + it( + "should return error if request includes 'if-unmodified-since' " + 'header and object has been modified', + done => { + const testDeleteRequest = new DummyRequest({ + bucketName, + namespace, + objectKey, + headers: { 'if-unmodified-since': earlyDate }, + url: `/${bucketName}/${objectKey}`, + }); + objectDelete(authInfo, testDeleteRequest, log, err => { + assert.strictEqual(err.is.PreconditionFailed, true); + done(); + }); + }, + ); - it('should delete an object with \'if-unmodified-since\' header', - done => { + it("should delete an object with 'if-unmodified-since' header", done => { const testDeleteRequest = new DummyRequest({ bucketName, namespace, @@ -331,31 +368,31 @@ describe('objectDelete API', () => { }); objectDelete(authInfo, testDeleteRequest, log, err => { assert.strictEqual(err, null); - objectGet(authInfo, testGetObjectRequest, false, log, - err => { + objectGet(authInfo, testGetObjectRequest, false, log, err => { assert.strictEqual(err.is.NoSuchKey, true); done(); }); }); }); - it('should return error if request includes \'if-modified-since\' ' + - 'header and object has not been modified', done => { - const testDeleteRequest = new DummyRequest({ - bucketName, - namespace, - objectKey, - headers: { 'if-modified-since': lateDate }, - url: `/${bucketName}/${objectKey}`, - }); - objectDelete(authInfo, testDeleteRequest, log, err => { - assert.strictEqual(err.is.NotModified, true); - done(); - }); - }); + it( + "should return error if request includes 'if-modified-since' " + 'header and object has not been modified', + done => { + const testDeleteRequest = new DummyRequest({ + bucketName, + namespace, + objectKey, + headers: { 'if-modified-since': lateDate }, + url: `/${bucketName}/${objectKey}`, + }); + objectDelete(authInfo, testDeleteRequest, log, err => { + assert.strictEqual(err.is.NotModified, true); + done(); + }); + }, + ); - it('should delete an object with \'if-modified-since\' header', - done => { + it("should delete an object with 'if-modified-since' header", done => { const testDeleteRequest = new DummyRequest({ bucketName, namespace, @@ -365,8 +402,7 @@ describe('objectDelete API', () => { }); objectDelete(authInfo, testDeleteRequest, log, err => { assert.strictEqual(err, null); - objectGet(authInfo, testGetObjectRequest, false, log, - err => { + objectGet(authInfo, testGetObjectRequest, false, log, err => { assert.strictEqual(err.is.NoSuchKey, true); done(); }); @@ -380,13 +416,16 @@ describe('objectDelete API with versioning', () => { beforeEach(() => { cleanup(); - testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey, - headers: {}, - url: `/${bucketName}/${objectKey}`, - }, postBody); + testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey, + headers: {}, + url: `/${bucketName}/${objectKey}`, + }, + postBody, + ); sinon.stub(services, 'deleteObject').callsFake(originalDeleteObject); sinon.spy(metadataswitch, 'putObjectMD'); @@ -412,27 +451,44 @@ describe('objectDelete API with versioning', () => { }); it('should upgrade master-only document to a version document when storing a delete marker version', done => { - async.series([ - next => bucketPut(authInfo, testBucketPutRequest, log, next), - next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), - next => bucketPutVersioning(authInfo, enableVersioningRequest, log, next), - next => objectDelete(authInfo, testDeleteRequest, log, next), - async () => { - const calls = metadataswitch.putObjectMD.getCalls(); - sinon.assert.calledWith(calls[calls.length - 2], - bucketName, objectKey, sinon.match({ - versionId: sinon.match.truthy, - isNull: true, - originOp: 's3:StoreNullVersion', - }), any, any, any); - }, - async () => { - // New version document (delete marker) was created with the right originOp. - sinon.assert.calledWith(metadataswitch.putObjectMD.lastCall, - bucketName, objectKey, sinon.match({ - _data: { originOp: 's3:ObjectRemoved:DeleteMarkerCreated' }, - }), any, any, any); - }, - ], done); + async.series( + [ + next => bucketPut(authInfo, testBucketPutRequest, log, next), + next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), + next => bucketPutVersioning(authInfo, enableVersioningRequest, log, next), + next => objectDelete(authInfo, testDeleteRequest, log, next), + async () => { + const calls = metadataswitch.putObjectMD.getCalls(); + sinon.assert.calledWith( + calls[calls.length - 2], + bucketName, + objectKey, + sinon.match({ + versionId: sinon.match.truthy, + isNull: true, + originOp: 's3:StoreNullVersion', + }), + any, + any, + any, + ); + }, + async () => { + // New version document (delete marker) was created with the right originOp. + sinon.assert.calledWith( + metadataswitch.putObjectMD.lastCall, + bucketName, + objectKey, + sinon.match({ + _data: { originOp: 's3:ObjectRemoved:DeleteMarkerCreated' }, + }), + any, + any, + any, + ); + }, + ], + done, + ); }); }); diff --git a/tests/unit/api/objectDeleteTagging.js b/tests/unit/api/objectDeleteTagging.js index 66ed5a11a2..4e9d751795 100644 --- a/tests/unit/api/objectDeleteTagging.js +++ b/tests/unit/api/objectDeleteTagging.js @@ -6,10 +6,7 @@ const objectPut = require('../../../lib/api/objectPut'); const objectPutTagging = require('../../../lib/api/objectPutTagging'); const objectDeleteTagging = require('../../../lib/api/objectDeleteTagging'); const metadata = require('../../../lib/metadata/wrapper'); -const { cleanup, - DummyRequestLogger, - makeAuthInfo, - TaggingConfigTester } = require('../helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo, TaggingConfigTester } = require('../helpers'); const DummyRequest = require('../DummyRequest'); const log = new DummyRequestLogger(); @@ -25,13 +22,16 @@ const testBucketPutRequest = { actionImplicitDenies: false, }; -const testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: {}, - url: `/${bucketName}/${objectName}`, -}, postBody); +const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: {}, + url: `/${bucketName}/${objectName}`, + }, + postBody, +); describe('deleteObjectTagging API', () => { beforeEach(done => { @@ -40,8 +40,7 @@ describe('deleteObjectTagging API', () => { if (err) { return done(err); } - return objectPut(authInfo, testPutObjectRequest, undefined, log, - done); + return objectPut(authInfo, testPutObjectRequest, undefined, log, done); }); }); @@ -49,22 +48,20 @@ describe('deleteObjectTagging API', () => { it('should delete tag set and update originOp', done => { const taggingUtil = new TaggingConfigTester(); - const testObjectPutTaggingRequest = taggingUtil - .createObjectTaggingRequest('PUT', bucketName, objectName); - const testObjectDeleteTaggingRequest = taggingUtil - .createObjectTaggingRequest('DELETE', bucketName, objectName); - async.waterfall([ - next => objectPutTagging(authInfo, testObjectPutTaggingRequest, log, - err => next(err)), - next => objectDeleteTagging(authInfo, - testObjectDeleteTaggingRequest, log, err => next(err)), - next => metadata.getObjectMD(bucketName, objectName, {}, log, - (err, objectMD) => next(err, objectMD)), - ], (err, objectMD) => { - const uploadedTags = objectMD.tags; - assert.deepStrictEqual(uploadedTags, {}); - assert.strictEqual(objectMD.originOp, 's3:ObjectTagging:Delete'); - return done(); - }); + const testObjectPutTaggingRequest = taggingUtil.createObjectTaggingRequest('PUT', bucketName, objectName); + const testObjectDeleteTaggingRequest = taggingUtil.createObjectTaggingRequest('DELETE', bucketName, objectName); + async.waterfall( + [ + next => objectPutTagging(authInfo, testObjectPutTaggingRequest, log, err => next(err)), + next => objectDeleteTagging(authInfo, testObjectDeleteTaggingRequest, log, err => next(err)), + next => metadata.getObjectMD(bucketName, objectName, {}, log, (err, objectMD) => next(err, objectMD)), + ], + (err, objectMD) => { + const uploadedTags = objectMD.tags; + assert.deepStrictEqual(uploadedTags, {}); + assert.strictEqual(objectMD.originOp, 's3:ObjectTagging:Delete'); + return done(); + }, + ); }); }); diff --git a/tests/unit/api/objectGet.js b/tests/unit/api/objectGet.js index 261bc149e0..91989c7310 100644 --- a/tests/unit/api/objectGet.js +++ b/tests/unit/api/objectGet.js @@ -5,11 +5,9 @@ const { parseString } = require('xml2js'); const { bucketPut } = require('../../../lib/api/bucketPut'); const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../helpers'); -const completeMultipartUpload - = require('../../../lib/api/completeMultipartUpload'); +const completeMultipartUpload = require('../../../lib/api/completeMultipartUpload'); const DummyRequest = require('../DummyRequest'); -const initiateMultipartUpload - = require('../../../lib/api/initiateMultipartUpload'); +const initiateMultipartUpload = require('../../../lib/api/initiateMultipartUpload'); const objectPut = require('../../../lib/api/objectPut'); const objectGet = require('../../../lib/api/objectGet'); const objectPutPart = require('../../../lib/api/objectPutPart'); @@ -29,17 +27,20 @@ describe('objectGet API', () => { beforeEach(() => { cleanup(); - testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { - 'x-amz-meta-test': 'some metadata', - 'content-length': '12', + testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { + 'x-amz-meta-test': 'some metadata', + 'content-length': '12', + }, + parsedContentLength: 12, + url: `/${bucketName}/${objectName}`, }, - parsedContentLength: 12, - url: `/${bucketName}/${objectName}`, - }, postBody); + postBody, + ); }); const correctMD5 = 'be747eb4b75517bf6b3cf7c5fbb62f3a'; @@ -63,19 +64,14 @@ describe('objectGet API', () => { it('should get the object metadata', done => { bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, - log, (err, resHeaders) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectGet(authInfo, testGetRequest, false, - log, (err, result, responseMetaHeaders) => { - assert.strictEqual( - responseMetaHeaders[userMetadataKey], - userMetadataValue); - assert.strictEqual(responseMetaHeaders.ETag, - `"${correctMD5}"`); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectGet(authInfo, testGetRequest, false, log, (err, result, responseMetaHeaders) => { + assert.strictEqual(responseMetaHeaders[userMetadataKey], userMetadataValue); + assert.strictEqual(responseMetaHeaders.ETag, `"${correctMD5}"`); + done(); }); + }); }); }); @@ -83,25 +79,29 @@ describe('objectGet API', () => { bucketName, namespace, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-bucket-object-lock-enabled': 'true', }, url: `/${bucketName}`, actionImplicitDenies: false, }; - const createPutDummyRetention = (date, mode) => new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { - 'x-amz-object-lock-retain-until-date': date, - 'x-amz-object-lock-mode': mode, - 'content-length': '12', - }, - parsedContentLength: 12, - url: `/${bucketName}/${objectName}`, - }, postBody); + const createPutDummyRetention = (date, mode) => + new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { + 'x-amz-object-lock-retain-until-date': date, + 'x-amz-object-lock-mode': mode, + 'content-length': '12', + }, + parsedContentLength: 12, + url: `/${bucketName}/${objectName}`, + }, + postBody, + ); const threeDaysMilliSecs = 3 * 24 * 60 * 60 * 1000; const testDate = new Date(Date.now() + threeDaysMilliSecs).toISOString(); @@ -109,204 +109,203 @@ describe('objectGet API', () => { it('should get the object metadata with valid retention info', done => { bucketPut(authInfo, testPutBucketRequestObjectLock, log, () => { const request = createPutDummyRetention(testDate, 'GOVERNANCE'); - objectPut(authInfo, request, undefined, - log, (err, headers) => { + objectPut(authInfo, request, undefined, log, (err, headers) => { + assert.ifError(err); + assert.strictEqual(headers.ETag, `"${correctMD5}"`); + const req = testGetRequest; + objectGet(authInfo, req, false, log, (err, r, headers) => { assert.ifError(err); + assert.strictEqual(headers['x-amz-object-lock-retain-until-date'], testDate); + assert.strictEqual(headers['x-amz-object-lock-mode'], 'GOVERNANCE'); assert.strictEqual(headers.ETag, `"${correctMD5}"`); - const req = testGetRequest; - objectGet(authInfo, req, false, log, (err, r, headers) => { - assert.ifError(err); - assert.strictEqual( - headers['x-amz-object-lock-retain-until-date'], - testDate); - assert.strictEqual( - headers['x-amz-object-lock-mode'], - 'GOVERNANCE'); - assert.strictEqual(headers.ETag, - `"${correctMD5}"`); - changeObjectLock([{ - bucket: bucketName, - key: objectName, - versionId: headers['x-amz-version-id'], - }], '', done); - }); + changeObjectLock( + [ + { + bucket: bucketName, + key: objectName, + versionId: headers['x-amz-version-id'], + }, + ], + '', + done, + ); }); + }); }); }); - const createPutDummyLegalHold = legalHold => new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { - 'x-amz-object-lock-legal-hold': legalHold, - 'content-length': '12', - }, - parsedContentLength: 12, - url: `/${bucketName}/${objectName}`, - }, postBody); + const createPutDummyLegalHold = legalHold => + new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { + 'x-amz-object-lock-legal-hold': legalHold, + 'content-length': '12', + }, + parsedContentLength: 12, + url: `/${bucketName}/${objectName}`, + }, + postBody, + ); const testStatuses = ['ON', 'OFF']; testStatuses.forEach(status => { it(`should get object metadata with legal hold ${status}`, done => { bucketPut(authInfo, testPutBucketRequestObjectLock, log, () => { const request = createPutDummyLegalHold(status); - objectPut(authInfo, request, undefined, log, - (err, resHeaders) => { + objectPut(authInfo, request, undefined, log, (err, resHeaders) => { + assert.ifError(err); + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectGet(authInfo, testGetRequest, false, log, (err, res, headers) => { assert.ifError(err); - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectGet(authInfo, testGetRequest, false, log, - (err, res, headers) => { - assert.ifError(err); - assert.strictEqual( - headers['x-amz-object-lock-legal-hold'], - status); - assert.strictEqual(headers.ETag, - `"${correctMD5}"`); - changeObjectLock([{ + assert.strictEqual(headers['x-amz-object-lock-legal-hold'], status); + assert.strictEqual(headers.ETag, `"${correctMD5}"`); + changeObjectLock( + [ + { bucket: bucketName, key: objectName, versionId: headers['x-amz-version-id'], - }], '', done); - }); + }, + ], + '', + done, + ); }); + }); }); }); }); const createPutDummyRetentionAndLegalHold = (date, mode, status) => - new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { - 'x-amz-object-lock-retain-until-date': date, - 'x-amz-object-lock-mode': mode, - 'x-amz-object-lock-legal-hold': status, - 'content-length': '12', + new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { + 'x-amz-object-lock-retain-until-date': date, + 'x-amz-object-lock-mode': mode, + 'x-amz-object-lock-legal-hold': status, + 'content-length': '12', + }, + parsedContentLength: 12, + url: `/${bucketName}/${objectName}`, }, - parsedContentLength: 12, - url: `/${bucketName}/${objectName}`, - }, postBody); + postBody, + ); - it('should get the object metadata with both retention and legal hold', - done => { - bucketPut(authInfo, testPutBucketRequestObjectLock, log, () => { - const request = createPutDummyRetentionAndLegalHold( - testDate, 'COMPLIANCE', 'ON'); - objectPut(authInfo, request, undefined, log, - (err, resHeaders) => { - assert.ifError(err); - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - const auth = authInfo; - const req = testGetRequest; - objectGet(auth, req, false, log, (err, r, headers) => { - assert.ifError(err); - assert.strictEqual( - headers['x-amz-object-lock-legal-hold'], - 'ON'); - assert.strictEqual( - headers['x-amz-object-lock-retain-until-date'], - testDate); - assert.strictEqual( - headers['x-amz-object-lock-mode'], - 'COMPLIANCE'); - assert.strictEqual(headers.ETag, - `"${correctMD5}"`); - done(); - }); - }); + it('should get the object metadata with both retention and legal hold', done => { + bucketPut(authInfo, testPutBucketRequestObjectLock, log, () => { + const request = createPutDummyRetentionAndLegalHold(testDate, 'COMPLIANCE', 'ON'); + objectPut(authInfo, request, undefined, log, (err, resHeaders) => { + assert.ifError(err); + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + const auth = authInfo; + const req = testGetRequest; + objectGet(auth, req, false, log, (err, r, headers) => { + assert.ifError(err); + assert.strictEqual(headers['x-amz-object-lock-legal-hold'], 'ON'); + assert.strictEqual(headers['x-amz-object-lock-retain-until-date'], testDate); + assert.strictEqual(headers['x-amz-object-lock-mode'], 'COMPLIANCE'); + assert.strictEqual(headers.ETag, `"${correctMD5}"`); + done(); + }); }); }); + }); it('should get the object data retrieval info', done => { bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, resHeaders) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectGet(authInfo, testGetRequest, false, log, - (err, dataGetInfo) => { - assert.deepStrictEqual(dataGetInfo, - [{ - key: 1, - start: 0, - size: 12, - dataStoreName: 'mem', - dataStoreETag: `1:${correctMD5}`, - }]); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectGet(authInfo, testGetRequest, false, log, (err, dataGetInfo) => { + assert.deepStrictEqual(dataGetInfo, [ + { + key: 1, + start: 0, + size: 12, + dataStoreName: 'mem', + dataStoreETag: `1:${correctMD5}`, + }, + ]); + done(); }); + }); }); }); - it('should get the object data retrieval info for an object put by MPU', - done => { - const partBody = Buffer.from('I am a part\n', 'utf8'); - const initiateRequest = { - bucketName, - namespace, - objectKey: objectName, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - url: `/${objectName}?uploads`, - actionImplicitDenies: false, - }; - async.waterfall([ + it('should get the object data retrieval info for an object put by MPU', done => { + const partBody = Buffer.from('I am a part\n', 'utf8'); + const initiateRequest = { + bucketName, + namespace, + objectKey: objectName, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: `/${objectName}?uploads`, + actionImplicitDenies: false, + }; + async.waterfall( + [ next => bucketPut(authInfo, testPutBucketRequest, log, next), - (corsHeaders, next) => initiateMultipartUpload(authInfo, - initiateRequest, log, next), + (corsHeaders, next) => initiateMultipartUpload(authInfo, initiateRequest, log, next), (result, corsHeaders, next) => parseString(result, next), (json, next) => { - const testUploadId = - json.InitiateMultipartUploadResult.UploadId[0]; + const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; const partHash = crypto.createHash('md5').update(partBody).digest('hex'); - const partRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { - 'host': `${bucketName}.s3.amazonaws.com`, - // Part (other than last part) must be at least 5MB - 'content-length': '5242880', - }, - parsedContentLength: 5242880, - url: `/${objectName}?partNumber=1&uploadId` + - `=${testUploadId}`, - query: { - partNumber: '1', - uploadId: testUploadId, + const partRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { + host: `${bucketName}.s3.amazonaws.com`, + // Part (other than last part) must be at least 5MB + 'content-length': '5242880', + }, + parsedContentLength: 5242880, + url: `/${objectName}?partNumber=1&uploadId` + `=${testUploadId}`, + query: { + partNumber: '1', + uploadId: testUploadId, + }, + partHash, }, - partHash, - }, partBody); + partBody, + ); objectPutPart(authInfo, partRequest, undefined, log, () => { next(null, testUploadId, partHash); }); }, (testUploadId, partHash, next) => { - const part2Request = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { - 'host': `${bucketName}.s3.amazonaws.com`, - 'content-length': '12', - }, - parsedContentLength: 12, - url: `/${objectName}?partNumber=2&uploadId=` + - `${testUploadId}`, - query: { - partNumber: '2', - uploadId: testUploadId, + const part2Request = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { + host: `${bucketName}.s3.amazonaws.com`, + 'content-length': '12', + }, + parsedContentLength: 12, + url: `/${objectName}?partNumber=2&uploadId=` + `${testUploadId}`, + query: { + partNumber: '2', + uploadId: testUploadId, + }, + partHash, }, - partHash, - }, partBody); - objectPutPart(authInfo, part2Request, undefined, - log, () => { - next(null, testUploadId, partHash); - }); + partBody, + ); + objectPutPart(authInfo, part2Request, undefined, log, () => { + next(null, testUploadId, partHash); + }); }, (testUploadId, partHash, next) => { - const completeBody = '' + + const completeBody = + '' + '' + '1' + `"${partHash}"` + @@ -327,19 +326,17 @@ describe('objectGet API', () => { post: completeBody, actionImplicitDenies: false, }; - completeMultipartUpload(authInfo, completeRequest, - log, err => { - next(err, partHash); - }); + completeMultipartUpload(authInfo, completeRequest, log, err => { + next(err, partHash); + }); }, ], (err, partHash) => { assert.ifError(err); - objectGet(authInfo, testGetRequest, false, log, - (err, dataGetInfo) => { + objectGet(authInfo, testGetRequest, false, log, (err, dataGetInfo) => { assert.ifError(err); - assert.deepStrictEqual(dataGetInfo, - [{ + assert.deepStrictEqual(dataGetInfo, [ + { key: 1, dataStoreName: 'mem', dataStoreETag: `1:${partHash}`, @@ -352,42 +349,42 @@ describe('objectGet API', () => { dataStoreETag: `2:${partHash}`, size: 12, start: 5242880, - }]); + }, + ]); done(); }); - }); - }); + }, + ); + }); it('should get a 0 bytes object', done => { const postBody = ''; const correctMD5 = 'd41d8cd98f00b204e9800998ecf8427e'; - const testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { - 'content-length': '0', - 'x-amz-meta-test': 'some metadata', + const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { + 'content-length': '0', + 'x-amz-meta-test': 'some metadata', + }, + parsedContentLength: 0, + url: `/${bucketName}/${objectName}`, + partHash: 'd41d8cd98f00b204e9800998ecf8427e', }, - parsedContentLength: 0, - url: `/${bucketName}/${objectName}`, - partHash: 'd41d8cd98f00b204e9800998ecf8427e', - }, postBody); + postBody, + ); bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, resHeaders) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectGet(authInfo, testGetRequest, false, - log, (err, result, responseMetaHeaders) => { - assert.strictEqual(result, null); - assert.strictEqual( - responseMetaHeaders[userMetadataKey], - userMetadataValue); - assert.strictEqual(responseMetaHeaders.ETag, - `"${correctMD5}"`); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectGet(authInfo, testGetRequest, false, log, (err, result, responseMetaHeaders) => { + assert.strictEqual(result, null); + assert.strictEqual(responseMetaHeaders[userMetadataKey], userMetadataValue); + assert.strictEqual(responseMetaHeaders.ETag, `"${correctMD5}"`); + done(); }); + }); }); }); @@ -493,28 +490,34 @@ describe('objectGet API', () => { }); }); - it('should reflect the restore header with ongoing-request=false and expiry-date set ' + - 'if the object is restored and not yet expired', done => { - const testGetRequest = { - bucketName, - namespace, - objectKey: objectName, - headers: {}, - url: `/${bucketName}/${objectName}`, - }; - mdColdHelper.putBucketMock(bucketName, null, () => { - const objectCustomMDFields = mdColdHelper.getRestoredObjectMD(); - const restoreInfo = objectCustomMDFields.getAmzRestore(); - mdColdHelper.putObjectMock(bucketName, objectName, objectCustomMDFields, () => { - objectGet(authInfo, testGetRequest, false, log, (err, res, headers) => { - assert.ifError(err); - assert.ok(res); - assert.strictEqual(headers['x-amz-storage-class'], mdColdHelper.defaultLocation); - const utcDate = new Date(restoreInfo.getExpiryDate()).toUTCString(); - assert.strictEqual(headers['x-amz-restore'], `ongoing-request="false", expiry-date="${utcDate}"`); - done(); + it( + 'should reflect the restore header with ongoing-request=false and expiry-date set ' + + 'if the object is restored and not yet expired', + done => { + const testGetRequest = { + bucketName, + namespace, + objectKey: objectName, + headers: {}, + url: `/${bucketName}/${objectName}`, + }; + mdColdHelper.putBucketMock(bucketName, null, () => { + const objectCustomMDFields = mdColdHelper.getRestoredObjectMD(); + const restoreInfo = objectCustomMDFields.getAmzRestore(); + mdColdHelper.putObjectMock(bucketName, objectName, objectCustomMDFields, () => { + objectGet(authInfo, testGetRequest, false, log, (err, res, headers) => { + assert.ifError(err); + assert.ok(res); + assert.strictEqual(headers['x-amz-storage-class'], mdColdHelper.defaultLocation); + const utcDate = new Date(restoreInfo.getExpiryDate()).toUTCString(); + assert.strictEqual( + headers['x-amz-restore'], + `ongoing-request="false", expiry-date="${utcDate}"`, + ); + done(); + }); }); }); - }); - }); + }, + ); }); diff --git a/tests/unit/api/objectGetACL.js b/tests/unit/api/objectGetACL.js index 5785a79a6a..3bf2538978 100644 --- a/tests/unit/api/objectGetACL.js +++ b/tests/unit/api/objectGetACL.js @@ -31,7 +31,7 @@ describe('objectGetACL API', () => { bucketName, namespace, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-acl': 'public-read-write', }, url: '/', @@ -48,36 +48,42 @@ describe('objectGetACL API', () => { }; it('should get a canned private ACL', done => { - const testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { 'x-amz-acl': 'private' }, - url: `/${bucketName}/${objectName}`, - post: postBody, - }, postBody); - async.waterfall([ - next => bucketPut(authInfo, testBucketPutRequest, log, next), - (corsHeaders, next) => objectPut(authInfo, testPutObjectRequest, - undefined, log, next), - (resHeaders, next) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectGetACL(authInfo, testGetACLRequest, log, next); + const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { 'x-amz-acl': 'private' }, + url: `/${bucketName}/${objectName}`, + post: postBody, }, - (result, corsHeaders, next) => parseString(result, next), - ], (err, result) => { - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Grantee[0] - .ID[0], canonicalID); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Permission[0], - 'FULL_CONTROL'); - done(); - }); + postBody, + ); + async.waterfall( + [ + next => bucketPut(authInfo, testBucketPutRequest, log, next), + (corsHeaders, next) => objectPut(authInfo, testPutObjectRequest, undefined, log, next), + (resHeaders, next) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectGetACL(authInfo, testGetACLRequest, log, next); + }, + (result, corsHeaders, next) => parseString(result, next), + ], + (err, result) => { + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Grantee[0].ID[0], + canonicalID, + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Permission[0], + 'FULL_CONTROL', + ); + done(); + }, + ); }); - it('should return an error if try to get an ACL ' + - 'for a nonexistent object', done => { + it('should return an error if try to get an ACL ' + 'for a nonexistent object', done => { bucketPut(authInfo, testBucketPutRequest, log, () => { objectGetACL(authInfo, testGetACLRequest, log, err => { assert.strictEqual(err.is.NoSuchKey, true); @@ -87,323 +93,343 @@ describe('objectGetACL API', () => { }); it('should get a canned public-read ACL', done => { - const testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { 'x-amz-acl': 'public-read' }, - url: `/${bucketName}/${objectName}`, - }, postBody); - async.waterfall([ - next => bucketPut(authInfo, testBucketPutRequest, log, next), - (corsHeaders, next) => objectPut(authInfo, testPutObjectRequest, - undefined, log, next), - (resHeaders, next) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectGetACL(authInfo, testGetACLRequest, log, next); + const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { 'x-amz-acl': 'public-read' }, + url: `/${bucketName}/${objectName}`, }, - (result, corsHeaders, next) => parseString(result, next), - ], (err, result) => { - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Grantee[0] - .ID[0], canonicalID); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Permission[0], - 'FULL_CONTROL'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[1].Grantee[0] - .URI[0], constants.publicId); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[1].Permission[0], 'READ'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[2], undefined); - done(); - }); + postBody, + ); + async.waterfall( + [ + next => bucketPut(authInfo, testBucketPutRequest, log, next), + (corsHeaders, next) => objectPut(authInfo, testPutObjectRequest, undefined, log, next), + (resHeaders, next) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectGetACL(authInfo, testGetACLRequest, log, next); + }, + (result, corsHeaders, next) => parseString(result, next), + ], + (err, result) => { + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Grantee[0].ID[0], + canonicalID, + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Permission[0], + 'FULL_CONTROL', + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[1].Grantee[0].URI[0], + constants.publicId, + ); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[1].Permission[0], 'READ'); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[2], undefined); + done(); + }, + ); }); it('should get a canned public-read-write ACL', done => { - const testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { 'x-amz-acl': 'public-read-write' }, - url: `/${bucketName}/${objectName}`, - }, postBody); - async.waterfall([ - next => bucketPut(authInfo, testBucketPutRequest, log, next), - (corsHeaders, next) => objectPut(authInfo, testPutObjectRequest, - undefined, log, next), - (resHeaders, next) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectGetACL(authInfo, testGetACLRequest, log, next); + const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { 'x-amz-acl': 'public-read-write' }, + url: `/${bucketName}/${objectName}`, }, - (result, corsHeaders, next) => parseString(result, next), - ], (err, result) => { - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Grantee[0] - .ID[0], canonicalID); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Permission[0], - 'FULL_CONTROL'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[1].Grantee[0] - .URI[0], constants.publicId); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[1].Permission[0], - 'READ'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[2].Grantee[0] - .URI[0], constants.publicId); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[2].Permission[0], - 'WRITE'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[3], undefined); - done(); - }); + postBody, + ); + async.waterfall( + [ + next => bucketPut(authInfo, testBucketPutRequest, log, next), + (corsHeaders, next) => objectPut(authInfo, testPutObjectRequest, undefined, log, next), + (resHeaders, next) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectGetACL(authInfo, testGetACLRequest, log, next); + }, + (result, corsHeaders, next) => parseString(result, next), + ], + (err, result) => { + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Grantee[0].ID[0], + canonicalID, + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Permission[0], + 'FULL_CONTROL', + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[1].Grantee[0].URI[0], + constants.publicId, + ); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[1].Permission[0], 'READ'); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[2].Grantee[0].URI[0], + constants.publicId, + ); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[2].Permission[0], 'WRITE'); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[3], undefined); + done(); + }, + ); }); it('should get a canned authenticated-read ACL', done => { - const testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { 'x-amz-acl': 'authenticated-read' }, - url: `/${bucketName}/${objectName}`, - }, postBody); - async.waterfall([ - next => bucketPut(authInfo, testBucketPutRequest, log, next), - (corsHeaders, next) => objectPut(authInfo, testPutObjectRequest, - undefined, log, next), - (resHeaders, next) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectGetACL(authInfo, testGetACLRequest, log, next); + const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { 'x-amz-acl': 'authenticated-read' }, + url: `/${bucketName}/${objectName}`, }, - (result, corsHeaders, next) => parseString(result, next), - ], (err, result) => { - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Grantee[0] - .ID[0], canonicalID); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Permission[0], - 'FULL_CONTROL'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[1].Grantee[0] - .URI[0], constants.allAuthedUsersId); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[1].Permission[0], - 'READ'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[2], - undefined); - done(); - }); + postBody, + ); + async.waterfall( + [ + next => bucketPut(authInfo, testBucketPutRequest, log, next), + (corsHeaders, next) => objectPut(authInfo, testPutObjectRequest, undefined, log, next), + (resHeaders, next) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectGetACL(authInfo, testGetACLRequest, log, next); + }, + (result, corsHeaders, next) => parseString(result, next), + ], + (err, result) => { + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Grantee[0].ID[0], + canonicalID, + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Permission[0], + 'FULL_CONTROL', + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[1].Grantee[0].URI[0], + constants.allAuthedUsersId, + ); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[1].Permission[0], 'READ'); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[2], undefined); + done(); + }, + ); }); it('should get a canned bucket-owner-read ACL', done => { - const testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { 'x-amz-acl': 'bucket-owner-read' }, - url: `/${bucketName}/${objectName}`, - post: postBody, - }, postBody); - async.waterfall([ - next => - bucketPut(otherAccountAuthInfo, testBucketPutRequest, - log, next), - (corsHeaders, next) => objectPut( - authInfo, testPutObjectRequest, undefined, log, next), - (resHeaders, next) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectGetACL(authInfo, testGetACLRequest, log, next); + const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { 'x-amz-acl': 'bucket-owner-read' }, + url: `/${bucketName}/${objectName}`, + post: postBody, }, - (result, corsHeaders, next) => parseString(result, next), - ], (err, result) => { - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Grantee[0] - .ID[0], canonicalID); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Permission[0], - 'FULL_CONTROL'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[1].Grantee[0] - .ID[0], otherAccountCanonicalID); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[1].Permission[0], - 'READ'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[2], undefined); - done(); - }); - }); - - it('should get a canned bucket-owner-full-control ACL', done => { - const testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { 'x-amz-acl': 'bucket-owner-full-control' }, - url: `/${bucketName}/${objectName}`, - calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', - }, postBody); - async.waterfall([ - next => - bucketPut(otherAccountAuthInfo, testBucketPutRequest, - log, next), - (corsHeaders, next) => objectPut(authInfo, testPutObjectRequest, - undefined, log, next), - (resHeaders, next) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectGetACL(authInfo, testGetACLRequest, log, next); + postBody, + ); + async.waterfall( + [ + next => bucketPut(otherAccountAuthInfo, testBucketPutRequest, log, next), + (corsHeaders, next) => objectPut(authInfo, testPutObjectRequest, undefined, log, next), + (resHeaders, next) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectGetACL(authInfo, testGetACLRequest, log, next); + }, + (result, corsHeaders, next) => parseString(result, next), + ], + (err, result) => { + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Grantee[0].ID[0], + canonicalID, + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Permission[0], + 'FULL_CONTROL', + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[1].Grantee[0].ID[0], + otherAccountCanonicalID, + ); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[1].Permission[0], 'READ'); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[2], undefined); + done(); }, - (result, corsHeaders, next) => parseString(result, next), - ], (err, result) => { - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Grantee[0] - .ID[0], canonicalID); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Permission[0], - 'FULL_CONTROL'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[1].Grantee[0] - .ID[0], otherAccountCanonicalID); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[1].Permission[0], - 'FULL_CONTROL'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[2], undefined); - done(); - }); + ); }); - it('should get specifically set ACLs', done => { - const testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { - 'x-amz-grant-full-control': - 'emailaddress="sampleaccount1@sampling.com"' + - ',emailaddress="sampleaccount2@sampling.com"', - 'x-amz-grant-read': `uri=${constants.allAuthedUsersId}`, - 'x-amz-grant-write': `uri=${constants.publicId}`, - 'x-amz-grant-read-acp': - 'id=79a59df900b949e55d96a1e698fbacedfd6e09d98eac' + - 'f8f8d5218e7cd47ef2be', - 'x-amz-grant-write-acp': - 'id=79a59df900b949e55d96a1e698fbacedfd6e09d98eac' + - 'f8f8d5218e7cd47ef2bf', + it('should get a canned bucket-owner-full-control ACL', done => { + const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { 'x-amz-acl': 'bucket-owner-full-control' }, + url: `/${bucketName}/${objectName}`, + calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', }, - url: `/${bucketName}/${objectName}`, - }, postBody); - async.waterfall([ - next => bucketPut(authInfo, testBucketPutRequest, - log, next), - (corsHeaders, next) => objectPut(authInfo, testPutObjectRequest, - undefined, log, next), - (resHeaders, next) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectGetACL(authInfo, testGetACLRequest, log, next); + postBody, + ); + async.waterfall( + [ + next => bucketPut(otherAccountAuthInfo, testBucketPutRequest, log, next), + (corsHeaders, next) => objectPut(authInfo, testPutObjectRequest, undefined, log, next), + (resHeaders, next) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectGetACL(authInfo, testGetACLRequest, log, next); + }, + (result, corsHeaders, next) => parseString(result, next), + ], + (err, result) => { + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Grantee[0].ID[0], + canonicalID, + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Permission[0], + 'FULL_CONTROL', + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[1].Grantee[0].ID[0], + otherAccountCanonicalID, + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[1].Permission[0], + 'FULL_CONTROL', + ); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[2], undefined); + done(); }, - (result, corsHeaders, next) => parseString(result, next), - ], (err, result) => { - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Grantee[0] - .ID[0], '79a59df900b949e55d96a1e698fbacedfd6e09d98' + - 'eacf8f8d5218e7cd47ef2be'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Grantee[0] - .DisplayName[0], 'sampleaccount1@sampling.com'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Permission[0], - 'FULL_CONTROL'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[1].Grantee[0] - .ID[0], '79a59df900b949e55d96a1e698fbacedfd6e09d98' + - 'eacf8f8d5218e7cd47ef2bf'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[1].Grantee[0] - .DisplayName[0], 'sampleaccount2@sampling.com'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[1].Permission[0], - 'FULL_CONTROL'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[2].Grantee[0] - .ID[0], '79a59df900b949e55d96a1e698fbacedfd6e09d98' + - 'eacf8f8d5218e7cd47ef2bf'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[2].Grantee[0] - .DisplayName[0], 'sampleaccount2@sampling.com'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[2].Permission[0], - 'WRITE_ACP'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[3].Grantee[0] - .ID[0], '79a59df900b949e55d96a1e698fbacedfd6e09d98' + - 'eacf8f8d5218e7cd47ef2be'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[3].Grantee[0] - .DisplayName[0], 'sampleaccount1@sampling.com'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[3].Permission[0], - 'READ_ACP'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[4].Grantee[0] - .URI[0], constants.allAuthedUsersId); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[4].Permission[0], - 'READ'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[5], - undefined); - done(); - }); + ); }); - const grantsByURI = [ - constants.publicId, - constants.allAuthedUsersId, - ]; - - grantsByURI.forEach(uri => { - it('should get all ACLs when predefined group - ' + - `${uri} is used for multiple grants`, done => { - const testPutObjectRequest = new DummyRequest({ + it('should get specifically set ACLs', done => { + const testPutObjectRequest = new DummyRequest( + { bucketName, namespace, objectKey: objectName, headers: { - 'x-amz-grant-full-control': `uri=${uri}`, - 'x-amz-grant-read': `uri=${uri}`, - 'x-amz-grant-read-acp': `uri=${uri}`, - 'x-amz-grant-write-acp': `uri=${uri}`, + 'x-amz-grant-full-control': + 'emailaddress="sampleaccount1@sampling.com"' + ',emailaddress="sampleaccount2@sampling.com"', + 'x-amz-grant-read': `uri=${constants.allAuthedUsersId}`, + 'x-amz-grant-write': `uri=${constants.publicId}`, + 'x-amz-grant-read-acp': 'id=79a59df900b949e55d96a1e698fbacedfd6e09d98eac' + 'f8f8d5218e7cd47ef2be', + 'x-amz-grant-write-acp': 'id=79a59df900b949e55d96a1e698fbacedfd6e09d98eac' + 'f8f8d5218e7cd47ef2bf', }, url: `/${bucketName}/${objectName}`, - }, postBody); - async.waterfall([ - next => bucketPut(authInfo, testBucketPutRequest, - log, next), - (corsHeaders, next) => objectPut(authInfo, - testPutObjectRequest, undefined, log, next), + }, + postBody, + ); + async.waterfall( + [ + next => bucketPut(authInfo, testBucketPutRequest, log, next), + (corsHeaders, next) => objectPut(authInfo, testPutObjectRequest, undefined, log, next), (resHeaders, next) => { assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); objectGetACL(authInfo, testGetACLRequest, log, next); }, (result, corsHeaders, next) => parseString(result, next), - ], (err, result) => { - assert.ifError(err); - const grants = - result.AccessControlPolicy.AccessControlList[0].Grant; - grants.forEach(grant => { - assert.strictEqual(grant.Permission.length, 1); - assert.strictEqual(grant.Grantee.length, 1); - assert.strictEqual(grant.Grantee[0].URI.length, 1); - assert.strictEqual(grant.Grantee[0].URI[0], `${uri}`); - }); + ], + (err, result) => { + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Grantee[0].ID[0], + '79a59df900b949e55d96a1e698fbacedfd6e09d98' + 'eacf8f8d5218e7cd47ef2be', + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Grantee[0].DisplayName[0], + 'sampleaccount1@sampling.com', + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Permission[0], + 'FULL_CONTROL', + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[1].Grantee[0].ID[0], + '79a59df900b949e55d96a1e698fbacedfd6e09d98' + 'eacf8f8d5218e7cd47ef2bf', + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[1].Grantee[0].DisplayName[0], + 'sampleaccount2@sampling.com', + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[1].Permission[0], + 'FULL_CONTROL', + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[2].Grantee[0].ID[0], + '79a59df900b949e55d96a1e698fbacedfd6e09d98' + 'eacf8f8d5218e7cd47ef2bf', + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[2].Grantee[0].DisplayName[0], + 'sampleaccount2@sampling.com', + ); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[2].Permission[0], 'WRITE_ACP'); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[3].Grantee[0].ID[0], + '79a59df900b949e55d96a1e698fbacedfd6e09d98' + 'eacf8f8d5218e7cd47ef2be', + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[3].Grantee[0].DisplayName[0], + 'sampleaccount1@sampling.com', + ); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[3].Permission[0], 'READ_ACP'); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[4].Grantee[0].URI[0], + constants.allAuthedUsersId, + ); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[4].Permission[0], 'READ'); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[5], undefined); done(); - }); + }, + ); + }); + + const grantsByURI = [constants.publicId, constants.allAuthedUsersId]; + + grantsByURI.forEach(uri => { + it('should get all ACLs when predefined group - ' + `${uri} is used for multiple grants`, done => { + const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { + 'x-amz-grant-full-control': `uri=${uri}`, + 'x-amz-grant-read': `uri=${uri}`, + 'x-amz-grant-read-acp': `uri=${uri}`, + 'x-amz-grant-write-acp': `uri=${uri}`, + }, + url: `/${bucketName}/${objectName}`, + }, + postBody, + ); + async.waterfall( + [ + next => bucketPut(authInfo, testBucketPutRequest, log, next), + (corsHeaders, next) => objectPut(authInfo, testPutObjectRequest, undefined, log, next), + (resHeaders, next) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectGetACL(authInfo, testGetACLRequest, log, next); + }, + (result, corsHeaders, next) => parseString(result, next), + ], + (err, result) => { + assert.ifError(err); + const grants = result.AccessControlPolicy.AccessControlList[0].Grant; + grants.forEach(grant => { + assert.strictEqual(grant.Permission.length, 1); + assert.strictEqual(grant.Grantee.length, 1); + assert.strictEqual(grant.Grantee[0].URI.length, 1); + assert.strictEqual(grant.Grantee[0].URI[0], `${uri}`); + }); + done(); + }, + ); }); }); }); diff --git a/tests/unit/api/objectGetAttributes.js b/tests/unit/api/objectGetAttributes.js index 4c951fbcdd..4c95c8c89f 100644 --- a/tests/unit/api/objectGetAttributes.js +++ b/tests/unit/api/objectGetAttributes.js @@ -23,15 +23,18 @@ const postBody = Buffer.from(body, 'utf8'); const expectedMD5 = 'fc3ff98e8c6a0d3087d515c0473f8677'; // Promisify helper for functions with non-standard callback signatures -const promisify = fn => (...args) => new Promise((resolve, reject) => { - fn(...args, (err, ...results) => { - if (err) { - reject(err); - } else { - resolve(results); - } - }); -}); +const promisify = + fn => + (...args) => + new Promise((resolve, reject) => { + fn(...args, (err, ...results) => { + if (err) { + reject(err); + } else { + resolve(results); + } + }); + }); const bucketPutAsync = promisify(bucketPut); const bucketPutVersioningAsync = promisify(bucketPutVersioning); @@ -104,7 +107,7 @@ describe('objectGetAttributes API', () => { assert.strictEqual( err.description, 'The x-amz-object-attributes header specifying the attributes ' + - 'to be retrieved is either missing or empty', + 'to be retrieved is either missing or empty', ); } }); @@ -174,12 +177,7 @@ describe('objectGetAttributes API', () => { }); it('should return all attributes', async () => { - const testGetRequest = createGetAttributesRequest([ - 'ETag', - 'ObjectParts', - 'StorageClass', - 'ObjectSize', - ]); + const testGetRequest = createGetAttributesRequest(['ETag', 'ObjectParts', 'StorageClass', 'ObjectSize']); const { xml, responseHeaders } = await objectGetAttributes(authInfo, testGetRequest, log); assert(xml, 'Response XML should be present'); @@ -296,8 +294,7 @@ describe('objectGetAttributes API with multipart upload', () => { completeParts.push(`${i}"${partHash}"`); } - const completeBody = - `${completeParts.join('')}`; + const completeBody = `${completeParts.join('')}`; const completeRequest = { bucketName, diff --git a/tests/unit/api/objectGetLegalHold.js b/tests/unit/api/objectGetLegalHold.js index 910cf12c74..cc2978795f 100644 --- a/tests/unit/api/objectGetLegalHold.js +++ b/tests/unit/api/objectGetLegalHold.js @@ -21,17 +21,19 @@ const bucketPutRequest = { actionImplicitDenies: false, }; -const putObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: {}, - url: `/${bucketName}/${objectName}`, -}, postBody); +const putObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: {}, + url: `/${bucketName}/${objectName}`, + }, + postBody, +); const objectLegalHoldXml = status => - '' + - `${status}`; + '' + `${status}`; const putObjectLegalHoldRequest = status => ({ bucketName, @@ -62,17 +64,17 @@ describe('getObjectLegalHold API', () => { afterEach(cleanup); it('should return InvalidRequest error', done => { - objectGetLegalHold(authInfo, getObjectLegalHoldRequest, log, - err => { - assert.strictEqual(err.is.InvalidRequest, true); - done(); - }); + objectGetLegalHold(authInfo, getObjectLegalHoldRequest, log, err => { + assert.strictEqual(err.is.InvalidRequest, true); + done(); + }); }); }); describe('with Object Lock enabled on bucket', () => { - const bucketObjectLockRequest = Object.assign({}, bucketPutRequest, - { headers: { 'x-amz-bucket-object-lock-enabled': 'true' } }); + const bucketObjectLockRequest = Object.assign({}, bucketPutRequest, { + headers: { 'x-amz-bucket-object-lock-enabled': 'true' }, + }); beforeEach(done => { bucketPut(authInfo, bucketObjectLockRequest, log, err => { @@ -83,42 +85,38 @@ describe('getObjectLegalHold API', () => { afterEach(cleanup); - it('should return NoSuchObjectLockConfiguration if no legal hold set', - done => { - objectGetLegalHold(authInfo, getObjectLegalHoldRequest, log, - err => { - assert.strictEqual(err.is.NoSuchObjectLockConfiguration, true); - done(); - }); + it('should return NoSuchObjectLockConfiguration if no legal hold set', done => { + objectGetLegalHold(authInfo, getObjectLegalHoldRequest, log, err => { + assert.strictEqual(err.is.NoSuchObjectLockConfiguration, true); + done(); }); + }); - it('should get an object\'s legal hold status when OFF', done => { + it("should get an object's legal hold status when OFF", done => { const status = 'OFF'; const request = putObjectLegalHoldRequest(status); objectPutLegalHold(authInfo, request, log, err => { assert.ifError(err); - objectGetLegalHold(authInfo, getObjectLegalHoldRequest, log, - (err, xml) => { - const expectedXml = objectLegalHoldXml(status); - assert.ifError(err); - assert.strictEqual(xml, expectedXml); - done(); - }); + objectGetLegalHold(authInfo, getObjectLegalHoldRequest, log, (err, xml) => { + const expectedXml = objectLegalHoldXml(status); + assert.ifError(err); + assert.strictEqual(xml, expectedXml); + done(); + }); }); }); - it('should get an object\'s legal hold status when ON', done => { + it("should get an object's legal hold status when ON", done => { const status = 'ON'; const request = putObjectLegalHoldRequest(status); objectPutLegalHold(authInfo, request, log, err => { assert.ifError(err); - objectGetLegalHold(authInfo, getObjectLegalHoldRequest, log, - (err, xml) => { - const expectedXml = objectLegalHoldXml(status); - assert.ifError(err); - assert.strictEqual(xml, expectedXml); - done(); - }); + objectGetLegalHold(authInfo, getObjectLegalHoldRequest, log, (err, xml) => { + const expectedXml = objectLegalHoldXml(status); + assert.ifError(err); + assert.strictEqual(xml, expectedXml); + done(); + }); }); }); }); diff --git a/tests/unit/api/objectGetRetention.js b/tests/unit/api/objectGetRetention.js index cd1481f98f..75818e91ce 100644 --- a/tests/unit/api/objectGetRetention.js +++ b/tests/unit/api/objectGetRetention.js @@ -24,15 +24,19 @@ const bucketPutRequest = { actionImplicitDenies: false, }; -const putObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: {}, - url: `/${bucketName}/${objectName}`, -}, postBody); +const putObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: {}, + url: `/${bucketName}/${objectName}`, + }, + postBody, +); -const objectRetentionXml = '' + 'GOVERNANCE' + `${date.toISOString()}` + @@ -74,8 +78,9 @@ describe('getObjectRetention API', () => { }); describe('with Object Lock enabled on bucket', () => { - const bucketObjLockRequest = Object.assign({}, bucketPutRequest, - { headers: { 'x-amz-bucket-object-lock-enabled': 'true' } }); + const bucketObjLockRequest = Object.assign({}, bucketPutRequest, { + headers: { 'x-amz-bucket-object-lock-enabled': 'true' }, + }); beforeEach(done => { bucketPut(authInfo, bucketObjLockRequest, log, err => { @@ -85,19 +90,17 @@ describe('getObjectRetention API', () => { }); afterEach(cleanup); - it('should return NoSuchObjectLockConfiguration if no retention set', - done => { + it('should return NoSuchObjectLockConfiguration if no retention set', done => { objectGetRetention(authInfo, getObjRetRequest, log, err => { assert.strictEqual(err.is.NoSuchObjectLockConfiguration, true); done(); }); }); - it('should get an object\'s retention info', done => { + it("should get an object's retention info", done => { objectPutRetention(authInfo, putObjRetRequest, log, err => { assert.ifError(err); - objectGetRetention(authInfo, getObjRetRequest, log, - (err, xml) => { + objectGetRetention(authInfo, getObjRetRequest, log, (err, xml) => { assert.ifError(err); assert.strictEqual(xml, objectRetentionXml); done(); diff --git a/tests/unit/api/objectGetTagging.js b/tests/unit/api/objectGetTagging.js index b099120fb2..753077b83d 100644 --- a/tests/unit/api/objectGetTagging.js +++ b/tests/unit/api/objectGetTagging.js @@ -4,11 +4,7 @@ const { bucketPut } = require('../../../lib/api/bucketPut'); const objectPut = require('../../../lib/api/objectPut'); const objectPutTagging = require('../../../lib/api/objectPutTagging'); const objectGetTagging = require('../../../lib/api/objectGetTagging'); -const { cleanup, - DummyRequestLogger, - makeAuthInfo, - TaggingConfigTester } - = require('../helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo, TaggingConfigTester } = require('../helpers'); const DummyRequest = require('../DummyRequest'); const log = new DummyRequestLogger(); @@ -24,13 +20,16 @@ const testBucketPutRequest = { actionImplicitDenies: false, }; -const testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: {}, - url: `/${bucketName}/${objectName}`, -}, postBody); +const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: {}, + url: `/${bucketName}/${objectName}`, + }, + postBody, +); describe('getObjectTagging API', () => { beforeEach(done => { @@ -39,8 +38,7 @@ describe('getObjectTagging API', () => { if (err) { return done(err); } - return objectPut(authInfo, testPutObjectRequest, undefined, log, - done); + return objectPut(authInfo, testPutObjectRequest, undefined, log, done); }); }); @@ -48,17 +46,14 @@ describe('getObjectTagging API', () => { it('should return tags resource', done => { const taggingUtil = new TaggingConfigTester(); - const testObjectPutTaggingRequest = taggingUtil - .createObjectTaggingRequest('PUT', bucketName, objectName); + const testObjectPutTaggingRequest = taggingUtil.createObjectTaggingRequest('PUT', bucketName, objectName); objectPutTagging(authInfo, testObjectPutTaggingRequest, log, err => { if (err) { process.stdout.write(`Err putting object tagging ${err}`); return done(err); } - const testObjectGetTaggingRequest = taggingUtil - .createObjectTaggingRequest('GET', bucketName, objectName); - return objectGetTagging(authInfo, testObjectGetTaggingRequest, log, - (err, xml) => { + const testObjectGetTaggingRequest = taggingUtil.createObjectTaggingRequest('GET', bucketName, objectName); + return objectGetTagging(authInfo, testObjectGetTaggingRequest, log, (err, xml) => { if (err) { process.stdout.write(`Err getting object tagging ${err}`); return done(err); diff --git a/tests/unit/api/objectHead.js b/tests/unit/api/objectHead.js index 511a39037c..7735ec5832 100644 --- a/tests/unit/api/objectHead.js +++ b/tests/unit/api/objectHead.js @@ -36,55 +36,61 @@ let testPutObjectRequest; describe('objectHead API', () => { beforeEach(() => { cleanup(); - testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { 'x-amz-meta-test': userMetadataValue }, - url: `/${bucketName}/${objectName}`, - calculatedHash: correctMD5, - }, postBody); + testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { 'x-amz-meta-test': userMetadataValue }, + url: `/${bucketName}/${objectName}`, + calculatedHash: correctMD5, + }, + postBody, + ); }); - it('should return NotModified if request header ' + - 'includes "if-modified-since" and object ' + - 'not modified since specified time', done => { - const testGetRequest = { - bucketName, - namespace, - objectKey: objectName, - headers: { 'if-modified-since': laterDate }, - url: `/${bucketName}/${objectName}`, - actionImplicitDenies: false, - }; + it( + 'should return NotModified if request header ' + + 'includes "if-modified-since" and object ' + + 'not modified since specified time', + done => { + const testGetRequest = { + bucketName, + namespace, + objectKey: objectName, + headers: { 'if-modified-since': laterDate }, + url: `/${bucketName}/${objectName}`, + actionImplicitDenies: false, + }; - bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, resHeaders) => { + bucketPut(authInfo, testPutBucketRequest, log, () => { + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); objectHead(authInfo, testGetRequest, log, err => { assert.strictEqual(err.is.NotModified, true); done(); }); }); - }); - }); + }); + }, + ); - it('should return PreconditionFailed if request header ' + - 'includes "if-unmodified-since" and object has ' + - 'been modified since specified time', done => { - const testGetRequest = { - bucketName, - namespace, - objectKey: objectName, - headers: { 'if-unmodified-since': earlierDate }, - url: `/${bucketName}/${objectName}`, - actionImplicitDenies: false, - }; + it( + 'should return PreconditionFailed if request header ' + + 'includes "if-unmodified-since" and object has ' + + 'been modified since specified time', + done => { + const testGetRequest = { + bucketName, + namespace, + objectKey: objectName, + headers: { 'if-unmodified-since': earlierDate }, + url: `/${bucketName}/${objectName}`, + actionImplicitDenies: false, + }; - bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, resHeaders) => { + bucketPut(authInfo, testPutBucketRequest, log, () => { + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { assert.ifError(err); assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); objectHead(authInfo, testGetRequest, log, err => { @@ -92,106 +98,118 @@ describe('objectHead API', () => { done(); }); }); - }); - }); + }); + }, + ); - it('should return PreconditionFailed if request header ' + - 'includes "if-match" and ETag of object ' + - 'does not match specified ETag', done => { - const testGetRequest = { - bucketName, - namespace, - objectKey: objectName, - headers: { 'if-match': incorrectMD5 }, - url: `/${bucketName}/${objectName}`, - actionImplicitDenies: false, - }; + it( + 'should return PreconditionFailed if request header ' + + 'includes "if-match" and ETag of object ' + + 'does not match specified ETag', + done => { + const testGetRequest = { + bucketName, + namespace, + objectKey: objectName, + headers: { 'if-match': incorrectMD5 }, + url: `/${bucketName}/${objectName}`, + actionImplicitDenies: false, + }; - bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, resHeaders) => { + bucketPut(authInfo, testPutBucketRequest, log, () => { + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); objectHead(authInfo, testGetRequest, log, err => { assert.strictEqual(err.is.PreconditionFailed, true); done(); }); }); - }); - }); + }); + }, + ); - it('should return NotModified if request header ' + - 'includes "if-none-match" and ETag of object does ' + - 'match specified ETag', done => { - const testGetRequest = { - bucketName, - namespace, - objectKey: objectName, - headers: { 'if-none-match': correctMD5 }, - url: `/${bucketName}/${objectName}`, - actionImplicitDenies: false, - }; + it( + 'should return NotModified if request header ' + + 'includes "if-none-match" and ETag of object does ' + + 'match specified ETag', + done => { + const testGetRequest = { + bucketName, + namespace, + objectKey: objectName, + headers: { 'if-none-match': correctMD5 }, + url: `/${bucketName}/${objectName}`, + actionImplicitDenies: false, + }; - bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, resHeaders) => { + bucketPut(authInfo, testPutBucketRequest, log, () => { + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); objectHead(authInfo, testGetRequest, log, err => { assert.strictEqual(err.is.NotModified, true); done(); }); }); - }); - }); + }); + }, + ); - it('should return Accept-Ranges header if request includes "Range" ' + - 'header with specified range bytes of an object', done => { - const testGetRequest = { - bucketName, - namespace, - objectKey: objectName, - headers: { range: 'bytes=1-9' }, - url: `/${bucketName}/${objectName}`, - actionImplicitDenies: false, - }; + it( + 'should return Accept-Ranges header if request includes "Range" ' + + 'header with specified range bytes of an object', + done => { + const testGetRequest = { + bucketName, + namespace, + objectKey: objectName, + headers: { range: 'bytes=1-9' }, + url: `/${bucketName}/${objectName}`, + actionImplicitDenies: false, + }; - bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, err => { - assert.strictEqual(err, null, `Error copying: ${err}`); - objectHead(authInfo, testGetRequest, log, (err, res) => { - assert.strictEqual(res['accept-ranges'], 'bytes'); - done(); + bucketPut(authInfo, testPutBucketRequest, log, () => { + objectPut(authInfo, testPutObjectRequest, undefined, log, err => { + assert.strictEqual(err, null, `Error copying: ${err}`); + objectHead(authInfo, testGetRequest, log, (err, res) => { + assert.strictEqual(res['accept-ranges'], 'bytes'); + done(); + }); }); }); - }); - }); + }, + ); - it('should return InvalidRequest error when both the Range header and ' + - 'the partNumber query parameter specified', done => { - const testGetRequest = { - bucketName, - namespace, - objectKey: objectName, - headers: { range: 'bytes=1-9' }, - url: `/${bucketName}/${objectName}`, - query: { - partNumber: '1', - }, - actionImplicitDenies: false, - }; + it( + 'should return InvalidRequest error when both the Range header and ' + + 'the partNumber query parameter specified', + done => { + const testGetRequest = { + bucketName, + namespace, + objectKey: objectName, + headers: { range: 'bytes=1-9' }, + url: `/${bucketName}/${objectName}`, + query: { + partNumber: '1', + }, + actionImplicitDenies: false, + }; - bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, err => { - assert.strictEqual(err, null, `Error objectPut: ${err}`); - objectHead(authInfo, testGetRequest, log, err => { - assert.strictEqual(err.is.InvalidRequest, true); - assert.strictEqual(err.description, - 'Cannot specify both Range header and ' + - 'partNumber query parameter.'); - done(); + bucketPut(authInfo, testPutBucketRequest, log, () => { + objectPut(authInfo, testPutObjectRequest, undefined, log, err => { + assert.strictEqual(err, null, `Error objectPut: ${err}`); + objectHead(authInfo, testGetRequest, log, err => { + assert.strictEqual(err.is.InvalidRequest, true); + assert.strictEqual( + err.description, + 'Cannot specify both Range header and ' + 'partNumber query parameter.', + ); + done(); + }); }); }); - }); - }); + }, + ); it('should return InvalidArgument error if partNumber is nan', done => { const testGetRequest = { @@ -218,27 +236,30 @@ describe('objectHead API', () => { }); }); - it('should not return Accept-Ranges header if request does not include ' + - '"Range" header with specified range bytes of an object', done => { - const testGetRequest = { - bucketName, - namespace, - objectKey: objectName, - headers: {}, - url: `/${bucketName}/${objectName}`, - actionImplicitDenies: false, - }; + it( + 'should not return Accept-Ranges header if request does not include ' + + '"Range" header with specified range bytes of an object', + done => { + const testGetRequest = { + bucketName, + namespace, + objectKey: objectName, + headers: {}, + url: `/${bucketName}/${objectName}`, + actionImplicitDenies: false, + }; - bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, err => { - assert.strictEqual(err, null, `Error objectPut: ${err}`); - objectHead(authInfo, testGetRequest, log, (err, res) => { - assert.strictEqual(res['accept-ranges'], undefined); - done(); + bucketPut(authInfo, testPutBucketRequest, log, () => { + objectPut(authInfo, testPutObjectRequest, undefined, log, err => { + assert.strictEqual(err, null, `Error objectPut: ${err}`); + objectHead(authInfo, testGetRequest, log, (err, res) => { + assert.strictEqual(res['accept-ranges'], undefined); + done(); + }); }); }); - }); - }); + }, + ); it('should get the object metadata', done => { const testGetRequest = { @@ -251,17 +272,14 @@ describe('objectHead API', () => { }; bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, resHeaders) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectHead(authInfo, testGetRequest, log, (err, res) => { - assert.strictEqual(res[userMetadataKey], - userMetadataValue); - assert - .strictEqual(res.ETag, `"${correctMD5}"`); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectHead(authInfo, testGetRequest, log, (err, res) => { + assert.strictEqual(res[userMetadataKey], userMetadataValue); + assert.strictEqual(res.ETag, `"${correctMD5}"`); + done(); }); + }); }); }); @@ -273,18 +291,21 @@ describe('objectHead API', () => { url: `/${bucketName}`, actionImplicitDenies: false, }; - const testPutObjectRequestLock = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { - 'x-amz-object-lock-retain-until-date': '2050-10-10', - 'x-amz-object-lock-mode': 'GOVERNANCE', - 'x-amz-object-lock-legal-hold': 'ON', + const testPutObjectRequestLock = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { + 'x-amz-object-lock-retain-until-date': '2050-10-10', + 'x-amz-object-lock-mode': 'GOVERNANCE', + 'x-amz-object-lock-legal-hold': 'ON', + }, + url: `/${bucketName}/${objectName}`, + calculatedHash: correctMD5, }, - url: `/${bucketName}/${objectName}`, - calculatedHash: correctMD5, - }, postBody); + postBody, + ); const testGetRequest = { bucketName, namespace, @@ -295,31 +316,30 @@ describe('objectHead API', () => { }; bucketPut(authInfo, testPutBucketRequestLock, log, () => { - objectPut(authInfo, testPutObjectRequestLock, undefined, log, - (err, resHeaders) => { + objectPut(authInfo, testPutObjectRequestLock, undefined, log, (err, resHeaders) => { + assert.ifError(err); + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectHead(authInfo, testGetRequest, log, (err, res) => { assert.ifError(err); - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectHead(authInfo, testGetRequest, log, (err, res) => { - assert.ifError(err); - const expectedDate = testPutObjectRequestLock - .headers['x-amz-object-lock-retain-until-date']; - const expectedMode = testPutObjectRequestLock - .headers['x-amz-object-lock-mode']; - assert.ifError(err); - assert.strictEqual( - res['x-amz-object-lock-retain-until-date'], - expectedDate); - assert.strictEqual(res['x-amz-object-lock-mode'], - expectedMode); - assert.strictEqual(res['x-amz-object-lock-legal-hold'], - 'ON'); - changeObjectLock([{ - bucket: bucketName, - key: objectName, - versionId: res['x-amz-version-id'], - }], '', done); - }); + const expectedDate = testPutObjectRequestLock.headers['x-amz-object-lock-retain-until-date']; + const expectedMode = testPutObjectRequestLock.headers['x-amz-object-lock-mode']; + assert.ifError(err); + assert.strictEqual(res['x-amz-object-lock-retain-until-date'], expectedDate); + assert.strictEqual(res['x-amz-object-lock-mode'], expectedMode); + assert.strictEqual(res['x-amz-object-lock-legal-hold'], 'ON'); + changeObjectLock( + [ + { + bucket: bucketName, + key: objectName, + versionId: res['x-amz-version-id'], + }, + ], + '', + done, + ); }); + }); }); }); @@ -393,37 +413,40 @@ describe('objectHead API', () => { }); }); - it('should reflect the restore header with ongoing-request=false and expiry-date set ' + - 'if the object is restored and not yet expired', done => { - const testGetRequest = { - bucketName, - namespace, - objectKey: objectName, - headers: {}, - url: `/${bucketName}/${objectName}`, - }; - mdColdHelper.putBucketMock(bucketName, null, () => { - const objectCustomMDFields = mdColdHelper.getRestoredObjectMD(); - mdColdHelper.putObjectMock(bucketName, objectName, objectCustomMDFields, () => { - objectHead(authInfo, testGetRequest, log, (err, res) => { - const restoreInfo = objectCustomMDFields.getAmzRestore(); - assert.strictEqual(res[userMetadataKey], userMetadataValue); - assert.strictEqual(res.ETag, `"${correctMD5}"`); - assert.strictEqual(res['x-amz-storage-class'], mdColdHelper.defaultLocation); - const utcDate = new Date(restoreInfo.getExpiryDate()).toUTCString(); - assert.strictEqual(res['x-amz-restore'], `ongoing-request="false", expiry-date="${utcDate}"`); - // Check we do not leak non-standard fields - assert.strictEqual(res['x-amz-scal-transition-in-progress'], undefined); - assert.strictEqual(res['x-amz-scal-archive-info'], undefined); - assert.strictEqual(res['x-amz-scal-restore-requested-at'], undefined); - assert.strictEqual(res['x-amz-scal-restore-completed-at'], undefined); - assert.strictEqual(res['x-amz-scal-restore-will-expire-at'], undefined); - assert.strictEqual(res['x-amz-scal-owner-id'], undefined); - done(); + it( + 'should reflect the restore header with ongoing-request=false and expiry-date set ' + + 'if the object is restored and not yet expired', + done => { + const testGetRequest = { + bucketName, + namespace, + objectKey: objectName, + headers: {}, + url: `/${bucketName}/${objectName}`, + }; + mdColdHelper.putBucketMock(bucketName, null, () => { + const objectCustomMDFields = mdColdHelper.getRestoredObjectMD(); + mdColdHelper.putObjectMock(bucketName, objectName, objectCustomMDFields, () => { + objectHead(authInfo, testGetRequest, log, (err, res) => { + const restoreInfo = objectCustomMDFields.getAmzRestore(); + assert.strictEqual(res[userMetadataKey], userMetadataValue); + assert.strictEqual(res.ETag, `"${correctMD5}"`); + assert.strictEqual(res['x-amz-storage-class'], mdColdHelper.defaultLocation); + const utcDate = new Date(restoreInfo.getExpiryDate()).toUTCString(); + assert.strictEqual(res['x-amz-restore'], `ongoing-request="false", expiry-date="${utcDate}"`); + // Check we do not leak non-standard fields + assert.strictEqual(res['x-amz-scal-transition-in-progress'], undefined); + assert.strictEqual(res['x-amz-scal-archive-info'], undefined); + assert.strictEqual(res['x-amz-scal-restore-requested-at'], undefined); + assert.strictEqual(res['x-amz-scal-restore-completed-at'], undefined); + assert.strictEqual(res['x-amz-scal-restore-will-expire-at'], undefined); + assert.strictEqual(res['x-amz-scal-owner-id'], undefined); + done(); + }); }); }); - }); - }); + }, + ); it('should report when transition in progress', done => { const testGetRequest = { @@ -465,8 +488,10 @@ describe('objectHead API', () => { objectHead(authInfo, testGetRequest, log, (err, res) => { assert.strictEqual(res['x-amz-meta-scal-s3-transition-in-progress'], true); assert.strictEqual(res['x-amz-scal-transition-in-progress'], true); - assert.strictEqual(res['x-amz-scal-transition-time'], - new Date(objectCustomMDFields.getTransitionTime()).toUTCString()); + assert.strictEqual( + res['x-amz-scal-transition-time'], + new Date(objectCustomMDFields.getTransitionTime()).toUTCString(), + ); assert.strictEqual(res['x-amz-scal-archive-info'], undefined); assert.strictEqual(res['x-amz-scal-owner-id'], mdColdHelper.defaultOwnerId); done(err); @@ -518,10 +543,11 @@ describe('objectHead API', () => { assert.strictEqual(res['x-amz-meta-scal-s3-transition-in-progress'], undefined); assert.strictEqual(res['x-amz-scal-transition-in-progress'], undefined); assert.strictEqual(res['x-amz-scal-archive-info'], '{"foo":0,"bar":"stuff"}'); - assert.strictEqual(res['x-amz-scal-restore-requested-at'], - new Date(archive.restoreRequestedAt).toUTCString()); - assert.strictEqual(res['x-amz-scal-restore-requested-days'], - archive.restoreRequestedDays); + assert.strictEqual( + res['x-amz-scal-restore-requested-at'], + new Date(archive.restoreRequestedAt).toUTCString(), + ); + assert.strictEqual(res['x-amz-scal-restore-requested-days'], archive.restoreRequestedDays); assert.strictEqual(res['x-amz-storage-class'], mdColdHelper.defaultLocation); assert.strictEqual(res['x-amz-scal-owner-id'], mdColdHelper.defaultOwnerId); done(err); @@ -548,14 +574,19 @@ describe('objectHead API', () => { assert.strictEqual(res['x-amz-meta-scal-s3-transition-in-progress'], undefined); assert.strictEqual(res['x-amz-scal-transition-in-progress'], undefined); assert.strictEqual(res['x-amz-scal-archive-info'], '{"foo":0,"bar":"stuff"}'); - assert.strictEqual(res['x-amz-scal-restore-requested-at'], - new Date(archive.restoreRequestedAt).toUTCString()); - assert.strictEqual(res['x-amz-scal-restore-requested-days'], - archive.restoreRequestedDays); - assert.strictEqual(res['x-amz-scal-restore-completed-at'], - new Date(archive.restoreCompletedAt).toUTCString()); - assert.strictEqual(res['x-amz-scal-restore-will-expire-at'], - new Date(archive.restoreWillExpireAt).toUTCString()); + assert.strictEqual( + res['x-amz-scal-restore-requested-at'], + new Date(archive.restoreRequestedAt).toUTCString(), + ); + assert.strictEqual(res['x-amz-scal-restore-requested-days'], archive.restoreRequestedDays); + assert.strictEqual( + res['x-amz-scal-restore-completed-at'], + new Date(archive.restoreCompletedAt).toUTCString(), + ); + assert.strictEqual( + res['x-amz-scal-restore-will-expire-at'], + new Date(archive.restoreWillExpireAt).toUTCString(), + ); assert.strictEqual(res['x-amz-scal-restore-etag'], mdColdHelper.restoredEtag); assert.strictEqual(res['x-amz-storage-class'], mdColdHelper.defaultLocation); assert.strictEqual(res['x-amz-scal-owner-id'], mdColdHelper.defaultOwnerId); @@ -570,30 +601,33 @@ describe('objectHead API', () => { name: 'should return content-length of 0 when requesting part 1 of empty object', partNumber: '1', expectedError: null, - expectedContentLength: 0 + expectedContentLength: 0, }, { name: 'should return InvalidRange error when requesting part > 1 of empty object', partNumber: '2', expectedError: 'InvalidRange', - expectedContentLength: undefined - } + expectedContentLength: undefined, + }, ].forEach(testCase => { it(testCase.name, done => { const emptyBody = ''; const emptyMD5 = 'd41d8cd98f00b204e9800998ecf8427e'; - const testPutEmptyObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { - 'content-length': '0', - 'x-amz-meta-test': userMetadataValue, + const testPutEmptyObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { + 'content-length': '0', + 'x-amz-meta-test': userMetadataValue, + }, + parsedContentLength: 0, + url: `/${bucketName}/${objectName}`, + calculatedHash: emptyMD5, }, - parsedContentLength: 0, - url: `/${bucketName}/${objectName}`, - calculatedHash: emptyMD5, - }, emptyBody); + emptyBody, + ); const testGetRequest = { bucketName, diff --git a/tests/unit/api/objectPut.js b/tests/unit/api/objectPut.js index 3f4c458d60..d771993812 100644 --- a/tests/unit/api/objectPut.js +++ b/tests/unit/api/objectPut.js @@ -10,25 +10,18 @@ const bucketPutACL = require('../../../lib/api/bucketPutACL'); const bucketPutVersioning = require('../../../lib/api/bucketPutVersioning'); const bucketPutPolicy = require('../../../lib/api/bucketPutPolicy'); const { parseTagFromQuery } = s3middleware.tagging; -const { cleanup, DummyRequestLogger, makeAuthInfo, versioningTestUtils } - = require('../helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo, versioningTestUtils } = require('../helpers'); const metadata = require('../metadataswitch'); const { data } = require('../../../lib/data/wrapper'); const objectPut = require('../../../lib/api/objectPut'); const { objectLockTestUtils } = require('../helpers'); const DummyRequest = require('../DummyRequest'); -const { - lastModifiedHeader, - maximumAllowedUploadSize, - objectLocationConstraintHeader, -} = require('../../../constants'); +const { lastModifiedHeader, maximumAllowedUploadSize, objectLocationConstraintHeader } = require('../../../constants'); const mpuUtils = require('../utils/mpuUtils'); const { fakeMetadataArchive } = require('../../functional/aws-node-sdk/test/utils/init'); const { config } = require('../../../lib/Config'); -const { - LOCATION_NAME_CRR, -} = require('../../constants'); +const { LOCATION_NAME_CRR } = require('../../constants'); const { ds } = storage.data.inMemory.datastore; @@ -52,7 +45,7 @@ const testPutBucketRequestLock = new DummyRequest({ bucketName, namespace, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-bucket-object-lock-enabled': 'true', }, url: '/', @@ -62,21 +55,18 @@ const originalputObjectMD = metadata.putObjectMD; const objectName = 'objectName'; let testPutObjectRequest; -const enableVersioningRequest = - versioningTestUtils.createBucketPutVersioningReq(bucketName, 'Enabled'); -const suspendVersioningRequest = - versioningTestUtils.createBucketPutVersioningReq(bucketName, 'Suspended'); +const enableVersioningRequest = versioningTestUtils.createBucketPutVersioningReq(bucketName, 'Enabled'); +const suspendVersioningRequest = versioningTestUtils.createBucketPutVersioningReq(bucketName, 'Suspended'); function testAuth(bucketOwner, authUser, bucketPutReq, log, cb) { bucketPut(bucketOwner, bucketPutReq, log, () => { bucketPutACL(bucketOwner, testPutBucketRequest, log, err => { assert.strictEqual(err, undefined); - objectPut(authUser, testPutObjectRequest, undefined, - log, (err, resHeaders) => { - assert.strictEqual(err, null); - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - cb(); - }); + objectPut(authUser, testPutObjectRequest, undefined, log, (err, resHeaders) => { + assert.strictEqual(err, null); + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + cb(); + }); }); }); } @@ -87,8 +77,7 @@ describe('parseTagFromQuery', () => { const allowedChar = '+- =._:/'; const tests = [ { tagging: 'key1=value1', result: { key1: 'value1' } }, - { tagging: `key1=${encodeURIComponent(allowedChar)}`, - result: { key1: allowedChar } }, + { tagging: `key1=${encodeURIComponent(allowedChar)}`, result: { key1: allowedChar } }, { tagging: 'key1=value1=value2', error: invalidArgument }, { tagging: '=value1', error: invalidArgument }, { tagging: 'key1%=value1', error: invalidArgument }, @@ -119,13 +108,16 @@ describe('objectPut API', () => { beforeEach(() => { cleanup(); sinon.spy(metadata, 'putObjectMD'); - testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - url: '/', - }, postBody); + testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: '/', + }, + postBody, + ); }); afterEach(() => { @@ -142,40 +134,38 @@ describe('objectPut API', () => { it('should return an error if user is not authorized', done => { const putAuthInfo = makeAuthInfo('accessKey2'); - bucketPut(putAuthInfo, testPutBucketRequest, - log, () => { - objectPut(authInfo, testPutObjectRequest, - undefined, log, err => { - assert.strictEqual(err.is.AccessDenied, true); - done(); - }); - }); - }); - - it('should return error if the upload size exceeds the ' + - 'maximum allowed upload size for a single PUT request', done => { - testPutObjectRequest.parsedContentLength = maximumAllowedUploadSize + 1; - bucketPut(authInfo, testPutBucketRequest, log, () => { + bucketPut(putAuthInfo, testPutBucketRequest, log, () => { objectPut(authInfo, testPutObjectRequest, undefined, log, err => { - assert.strictEqual(err.is.EntityTooLarge, true); + assert.strictEqual(err.is.AccessDenied, true); done(); }); }); }); + it( + 'should return error if the upload size exceeds the ' + 'maximum allowed upload size for a single PUT request', + done => { + testPutObjectRequest.parsedContentLength = maximumAllowedUploadSize + 1; + bucketPut(authInfo, testPutBucketRequest, log, () => { + objectPut(authInfo, testPutObjectRequest, undefined, log, err => { + assert.strictEqual(err.is.EntityTooLarge, true); + done(); + }); + }); + }, + ); + it('should put object if user has FULL_CONTROL grant on bucket', done => { const bucketOwner = makeAuthInfo('accessKey2'); const authUser = makeAuthInfo('accessKey3'); - testPutBucketRequest.headers['x-amz-grant-full-control'] = - `id=${authUser.getCanonicalID()}`; + testPutBucketRequest.headers['x-amz-grant-full-control'] = `id=${authUser.getCanonicalID()}`; testAuth(bucketOwner, authUser, testPutBucketRequest, log, done); }); it('should put object if user has WRITE grant on bucket', done => { const bucketOwner = makeAuthInfo('accessKey2'); const authUser = makeAuthInfo('accessKey3'); - testPutBucketRequest.headers['x-amz-grant-write'] = - `id=${authUser.getCanonicalID()}`; + testPutBucketRequest.headers['x-amz-grant-write'] = `id=${authUser.getCanonicalID()}`; testAuth(bucketOwner, authUser, testPutBucketRequest, log, done); }); @@ -189,60 +179,61 @@ describe('objectPut API', () => { }); it('should successfully put an object', done => { - const testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: {}, - url: `/${bucketName}/${objectName}`, - calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', - }, postBody); + const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: {}, + url: `/${bucketName}/${objectName}`, + calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', + }, + postBody, + ); bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, resHeaders) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - metadata.getObjectMD(bucketName, objectName, - {}, log, (err, md) => { - assert(md); - assert - .strictEqual(md['content-md5'], correctMD5); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => { + assert(md); + assert.strictEqual(md['content-md5'], correctMD5); + done(); }); + }); }); }); const mockModes = ['GOVERNANCE', 'COMPLIANCE']; mockModes.forEach(mockMode => { it(`should put an object with valid date & ${mockMode} mode`, done => { - const testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { - 'x-amz-object-lock-retain-until-date': mockDate, - 'x-amz-object-lock-mode': mockMode, + const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { + 'x-amz-object-lock-retain-until-date': mockDate, + 'x-amz-object-lock-mode': mockMode, + }, + url: `/${bucketName}/${objectName}`, + calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', }, - url: `/${bucketName}/${objectName}`, - calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', - }, postBody); + postBody, + ); bucketPut(authInfo, testPutBucketRequestLock, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, headers) => { + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, headers) => { + assert.ifError(err); + assert.strictEqual(headers.ETag, `"${correctMD5}"`); + metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => { + const mode = md.retentionMode; + const retainUntilDate = md.retentionDate; assert.ifError(err); - assert.strictEqual(headers.ETag, `"${correctMD5}"`); - metadata.getObjectMD(bucketName, objectName, {}, log, - (err, md) => { - const mode = md.retentionMode; - const retainUntilDate = md.retentionDate; - assert.ifError(err); - assert(md); - assert.strictEqual(mode, mockMode); - assert.strictEqual(retainUntilDate, mockDate); - done(); - }); + assert(md); + assert.strictEqual(mode, mockMode); + assert.strictEqual(retainUntilDate, mockDate); + done(); }); + }); }); }); }); @@ -261,311 +252,323 @@ describe('objectPut API', () => { ]; testObjectLockConfigs.forEach(lockConfig => { const { testMode, type, val } = lockConfig; - it('should put an object with default retention if object does not ' + - 'have retention configuration but bucket has', done => { - const testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: {}, - url: `/${bucketName}/${objectName}`, - calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', - }, postBody); + it( + 'should put an object with default retention if object does not ' + + 'have retention configuration but bucket has', + done => { + const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: {}, + url: `/${bucketName}/${objectName}`, + calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', + }, + postBody, + ); - const testObjLockRequest = { - bucketName, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - post: objectLockTestUtils.generateXml(testMode, val, type), - }; + const testObjLockRequest = { + bucketName, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + post: objectLockTestUtils.generateXml(testMode, val, type), + }; - bucketPut(authInfo, testPutBucketRequestLock, log, () => { - bucketPutObjectLock(authInfo, testObjLockRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, headers) => { + bucketPut(authInfo, testPutBucketRequestLock, log, () => { + bucketPutObjectLock(authInfo, testObjLockRequest, log, () => { + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, headers) => { assert.ifError(err); assert.strictEqual(headers.ETag, `"${correctMD5}"`); - metadata.getObjectMD(bucketName, objectName, {}, - log, (err, md) => { - assert.ifError(err); - - const mode = md.retentionMode; - assert.strictEqual(mode, testMode); - - const retainDate = moment(md.retentionDate); - const days = type === 'Days' ? val : val * 365; - const { scaledMsPerDay } = config.getTimeOptions(); - const date = moment().add(days * scaledMsPerDay, 'ms'); - const dateDiff = retainDate.diff(date, 'ms'); - assert.ok(dateDiff < 10); - - done(); - }); + metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => { + assert.ifError(err); + + const mode = md.retentionMode; + assert.strictEqual(mode, testMode); + + const retainDate = moment(md.retentionDate); + const days = type === 'Days' ? val : val * 365; + const { scaledMsPerDay } = config.getTimeOptions(); + const date = moment().add(days * scaledMsPerDay, 'ms'); + const dateDiff = retainDate.diff(date, 'ms'); + assert.ok(dateDiff < 10); + + done(); + }); }); + }); }); - }); - }); + }, + ); }); it('should successfully put an object with legal hold ON', done => { - const request = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { - 'x-amz-object-lock-legal-hold': 'ON', + const request = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { + 'x-amz-object-lock-legal-hold': 'ON', + }, + url: `/${bucketName}/${objectName}`, + calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', }, - url: `/${bucketName}/${objectName}`, - calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', - }, postBody); + postBody, + ); bucketPut(authInfo, testPutBucketRequestLock, log, () => { objectPut(authInfo, request, undefined, log, (err, headers) => { assert.ifError(err); assert.strictEqual(headers.ETag, `"${correctMD5}"`); - metadata.getObjectMD(bucketName, objectName, {}, log, - (err, md) => { - assert.ifError(err); - assert.strictEqual(md.legalHold, true); - done(); - }); + metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => { + assert.ifError(err); + assert.strictEqual(md.legalHold, true); + done(); + }); }); }); }); it('should successfully put an object with legal hold OFF', done => { - const request = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { - 'x-amz-object-lock-legal-hold': 'OFF', + const request = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { + 'x-amz-object-lock-legal-hold': 'OFF', + }, + url: `/${bucketName}/${objectName}`, + calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', }, - url: `/${bucketName}/${objectName}`, - calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', - }, postBody); + postBody, + ); bucketPut(authInfo, testPutBucketRequestLock, log, () => { objectPut(authInfo, request, undefined, log, (err, headers) => { assert.ifError(err); assert.strictEqual(headers.ETag, `"${correctMD5}"`); - metadata.getObjectMD(bucketName, objectName, {}, log, - (err, md) => { - assert.ifError(err); - assert(md); - assert.strictEqual(md.legalHold, false); - done(); - }); + metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => { + assert.ifError(err); + assert(md); + assert.strictEqual(md.legalHold, false); + done(); + }); }); }); }); it('should successfully put an object with user metadata', done => { - const testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { - // Note that Node will collapse common headers into one - // (e.g. "x-amz-meta-test: hi" and "x-amz-meta-test: - // there" becomes "x-amz-meta-test: hi, there") - // Here we are not going through an actual http - // request so will not collapse properly. - 'x-amz-meta-test': 'some metadata', - 'x-amz-meta-test2': 'some more metadata', - 'x-amz-meta-test3': 'even more metadata', + const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { + // Note that Node will collapse common headers into one + // (e.g. "x-amz-meta-test: hi" and "x-amz-meta-test: + // there" becomes "x-amz-meta-test: hi, there") + // Here we are not going through an actual http + // request so will not collapse properly. + 'x-amz-meta-test': 'some metadata', + 'x-amz-meta-test2': 'some more metadata', + 'x-amz-meta-test3': 'even more metadata', + }, + url: `/${bucketName}/${objectName}`, + calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', }, - url: `/${bucketName}/${objectName}`, - calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', - }, postBody); + postBody, + ); bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, resHeaders) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - metadata.getObjectMD(bucketName, objectName, {}, log, - (err, md) => { - assert(md); - assert.strictEqual(md['x-amz-meta-test'], - 'some metadata'); - assert.strictEqual(md['x-amz-meta-test2'], - 'some more metadata'); - assert.strictEqual(md['x-amz-meta-test3'], - 'even more metadata'); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => { + assert(md); + assert.strictEqual(md['x-amz-meta-test'], 'some metadata'); + assert.strictEqual(md['x-amz-meta-test2'], 'some more metadata'); + assert.strictEqual(md['x-amz-meta-test3'], 'even more metadata'); + done(); }); + }); }); }); it('If testingMode=true and the last-modified header is given, should set last-modified accordingly', done => { const imposedLastModified = '2024-07-19'; - const testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { - [lastModifiedHeader]: imposedLastModified, + const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { + [lastModifiedHeader]: imposedLastModified, + }, + url: `/${bucketName}/${objectName}`, + calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', }, - url: `/${bucketName}/${objectName}`, - calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', - }, postBody); + postBody, + ); bucketPut(authInfo, testPutBucketRequest, log, () => { config.testingMode = true; - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, resHeaders) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - metadata.getObjectMD(bucketName, objectName, {}, log, - (err, md) => { - assert(md); - - const lastModified = md['last-modified']; - const lastModifiedDate = lastModified.split('T')[0]; - // last-modified date should be the one set by the last-modified header - assert.strictEqual(lastModifiedDate, imposedLastModified); - - // The header should be removed after being treated. - assert(md[lastModifiedHeader] === undefined); - - config.testingMode = false; - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => { + assert(md); + + const lastModified = md['last-modified']; + const lastModifiedDate = lastModified.split('T')[0]; + // last-modified date should be the one set by the last-modified header + assert.strictEqual(lastModifiedDate, imposedLastModified); + + // The header should be removed after being treated. + assert(md[lastModifiedHeader] === undefined); + + config.testingMode = false; + done(); }); + }); }); }); it('should not take into acccount the last-modified header when testingMode=false', done => { const imposedLastModified = '2024-07-19'; - const testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { - 'x-amz-meta-x-scal-last-modified': imposedLastModified, + const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { + 'x-amz-meta-x-scal-last-modified': imposedLastModified, + }, + url: `/${bucketName}/${objectName}`, + calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', }, - url: `/${bucketName}/${objectName}`, - calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', - }, postBody); + postBody, + ); bucketPut(authInfo, testPutBucketRequest, log, () => { config.testingMode = false; - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, resHeaders) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - metadata.getObjectMD(bucketName, objectName, {}, log, - (err, md) => { - assert(md); - assert.strictEqual(md['x-amz-meta-x-scal-last-modified'], - imposedLastModified); - const lastModified = md['last-modified']; - const lastModifiedDate = lastModified.split('T')[0]; - const currentTs = new Date().toJSON(); - const currentDate = currentTs.split('T')[0]; - assert.strictEqual(lastModifiedDate, currentDate); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => { + assert(md); + assert.strictEqual(md['x-amz-meta-x-scal-last-modified'], imposedLastModified); + const lastModified = md['last-modified']; + const lastModifiedDate = lastModified.split('T')[0]; + const currentTs = new Date().toJSON(); + const currentDate = currentTs.split('T')[0]; + assert.strictEqual(lastModifiedDate, currentDate); + done(); }); + }); }); }); it('should put an object with user metadata but no data', done => { const postBody = ''; const correctMD5 = 'd41d8cd98f00b204e9800998ecf8427e'; - const testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { - 'content-length': '0', - 'x-amz-meta-test': 'some metadata', - 'x-amz-meta-test2': 'some more metadata', - 'x-amz-meta-test3': 'even more metadata', + const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { + 'content-length': '0', + 'x-amz-meta-test': 'some metadata', + 'x-amz-meta-test2': 'some more metadata', + 'x-amz-meta-test3': 'even more metadata', + }, + parsedContentLength: 0, + url: `/${bucketName}/${objectName}`, + calculatedHash: 'd41d8cd98f00b204e9800998ecf8427e', }, - parsedContentLength: 0, - url: `/${bucketName}/${objectName}`, - calculatedHash: 'd41d8cd98f00b204e9800998ecf8427e', - }, postBody); + postBody, + ); bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, resHeaders) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - assert.deepStrictEqual(ds, []); - metadata.getObjectMD(bucketName, objectName, {}, log, - (err, md) => { - assert(md); - assert.strictEqual(md.location, null); - assert.strictEqual(md['x-amz-meta-test'], - 'some metadata'); - assert.strictEqual(md['x-amz-meta-test2'], - 'some more metadata'); - assert.strictEqual(md['x-amz-meta-test3'], - 'even more metadata'); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + assert.deepStrictEqual(ds, []); + metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => { + assert(md); + assert.strictEqual(md.location, null); + assert.strictEqual(md['x-amz-meta-test'], 'some metadata'); + assert.strictEqual(md['x-amz-meta-test2'], 'some more metadata'); + assert.strictEqual(md['x-amz-meta-test3'], 'even more metadata'); + done(); }); + }); }); }); it('should not leave orphans in data when overwriting an object', done => { - const testPutObjectRequest2 = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: {}, - url: `/${bucketName}/${objectName}`, - }, Buffer.from('I am another body', 'utf8')); + const testPutObjectRequest2 = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: {}, + url: `/${bucketName}/${objectName}`, + }, + Buffer.from('I am another body', 'utf8'), + ); bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, - undefined, log, () => { - objectPut(authInfo, testPutObjectRequest2, undefined, - log, - () => { - // orphan objects don't get deleted - // until the next tick - // in memory - setImmediate(() => { - // Data store starts at index 1 - assert.strictEqual(ds[0], undefined); - assert.strictEqual(ds[1], undefined); - assert.deepStrictEqual(ds[2].value, - Buffer.from('I am another body', 'utf8')); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, () => { + objectPut(authInfo, testPutObjectRequest2, undefined, log, () => { + // orphan objects don't get deleted + // until the next tick + // in memory + setImmediate(() => { + // Data store starts at index 1 + assert.strictEqual(ds[0], undefined); + assert.strictEqual(ds[1], undefined); + assert.deepStrictEqual(ds[2].value, Buffer.from('I am another body', 'utf8')); + done(); }); }); + }); }); }); it('should not leave orphans in data when overwriting an multipart upload object', done => { bucketPut(authInfo, testPutBucketRequest, log, () => { - mpuUtils.createMPU(namespace, bucketName, objectName, log, - (err, testUploadId) => { - objectPut(authInfo, testPutObjectRequest, undefined, log, err => { - assert.ifError(err); - sinon.assert.calledWith(metadata.putObjectMD, - any, any, any, sinon.match({ oldReplayId: testUploadId }), any, any); - done(); - }); + mpuUtils.createMPU(namespace, bucketName, objectName, log, (err, testUploadId) => { + objectPut(authInfo, testPutObjectRequest, undefined, log, err => { + assert.ifError(err); + sinon.assert.calledWith( + metadata.putObjectMD, + any, + any, + any, + sinon.match({ oldReplayId: testUploadId }), + any, + any, + ); + done(); }); + }); }); }); - it('should not put object with retention configuration if object lock ' + - 'is not enabled on the bucket', done => { - const testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { - 'x-amz-object-lock-retain-until-date': mockDate, - 'x-amz-object-lock-mode': 'GOVERNANCE', + it('should not put object with retention configuration if object lock ' + 'is not enabled on the bucket', done => { + const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { + 'x-amz-object-lock-retain-until-date': mockDate, + 'x-amz-object-lock-mode': 'GOVERNANCE', + }, + url: `/${bucketName}/${objectName}`, + calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', }, - url: `/${bucketName}/${objectName}`, - calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', - }, postBody); + postBody, + ); bucketPut(authInfo, testPutBucketRequest, log, () => { objectPut(authInfo, testPutObjectRequest, undefined, log, err => { @@ -577,233 +580,313 @@ describe('objectPut API', () => { }); it('should forward a 400 back to client on metadata 408 response', () => { - data.switch(new storage.data.MultipleBackendGateway({ - 'us-east-1': dataClient, - 'us-east-2': dataClient, - }, metadata, data.locStorageCheckFn)); + data.switch( + new storage.data.MultipleBackendGateway( + { + 'us-east-1': dataClient, + 'us-east-2': dataClient, + }, + metadata, + data.locStorageCheckFn, + ), + ); data.implName = 'multipleBackends'; const originalPut = data.client.put; - data.client.put = (hashedStream, valueSize, keyContext, backendInfo, log, cb) => - cb({ httpCode: 408 }); + data.client.put = (hashedStream, valueSize, keyContext, backendInfo, log, cb) => cb({ httpCode: 408 }); bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - err => { - assert.strictEqual(err.code, 400); - data.client.put = originalPut; - data.switch(dataClient); - data.implName = prevDataImplName; - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, err => { + assert.strictEqual(err.code, 400); + data.client.put = originalPut; + data.switch(dataClient); + data.implName = prevDataImplName; + }); }); }); it('should forward a 503 to the client for 4xx != 408', () => { - data.switch(new storage.data.MultipleBackendGateway({ - 'us-east-1': dataClient, - 'us-east-2': dataClient, - }, metadata, data.locStorageCheckFn)); + data.switch( + new storage.data.MultipleBackendGateway( + { + 'us-east-1': dataClient, + 'us-east-2': dataClient, + }, + metadata, + data.locStorageCheckFn, + ), + ); data.implName = 'multipleBackends'; const originalPut = data.client.put; - data.client.put = (hashedStream, valueSize, keyContext, backendInfo, log, cb) => - cb({ httpCode: 412 }); + data.client.put = (hashedStream, valueSize, keyContext, backendInfo, log, cb) => cb({ httpCode: 412 }); bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - err => { - assert.strictEqual(err.code, 503); - data.client.put = originalPut; - data.switch(dataClient); - data.implName = prevDataImplName; - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, err => { + assert.strictEqual(err.code, 503); + data.client.put = originalPut; + data.switch(dataClient); + data.implName = prevDataImplName; + }); }); }); it('should not put object with storage-class header not equal to STANDARD', done => { - const testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { - 'x-amz-storage-class': 'COLD', + const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { + 'x-amz-storage-class': 'COLD', + }, + url: `/${bucketName}/${objectName}`, + calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', }, - url: `/${bucketName}/${objectName}`, - calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', - }, postBody); + postBody, + ); bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - err => { - assert.strictEqual(err.is.InvalidStorageClass, true); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, err => { + assert.strictEqual(err.is.InvalidStorageClass, true); + done(); + }); }); }); it('should pass overheadField to metadata.putObjectMD for a non-versioned request', done => { - const testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: {}, - url: `/${bucketName}/${objectName}`, - contentMD5: correctMD5, - }, postBody); + const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: {}, + url: `/${bucketName}/${objectName}`, + contentMD5: correctMD5, + }, + postBody, + ); bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - err => { - assert.ifError(err); - sinon.assert.calledWith(metadata.putObjectMD.lastCall, - bucketName, objectName, any, sinon.match({ overheadField: sinon.match.array }), any, any); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, err => { + assert.ifError(err); + sinon.assert.calledWith( + metadata.putObjectMD.lastCall, + bucketName, + objectName, + any, + sinon.match({ overheadField: sinon.match.array }), + any, + any, + ); + done(); + }); }); }); it('should pass overheadField to metadata.putObjectMD for a versioned request', done => { - const testPutObjectRequest = versioningTestUtils - .createPutObjectRequest(bucketName, objectName, Buffer.from('I am another body', 'utf8')); + const testPutObjectRequest = versioningTestUtils.createPutObjectRequest( + bucketName, + objectName, + Buffer.from('I am another body', 'utf8'), + ); bucketPut(authInfo, testPutBucketRequest, log, () => { bucketPutVersioning(authInfo, enableVersioningRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - err => { - assert.ifError(err); - sinon.assert.calledWith(metadata.putObjectMD.lastCall, - bucketName, objectName, any, sinon.match({ overheadField: sinon.match.array }), any, any); - done(); - } - ); + objectPut(authInfo, testPutObjectRequest, undefined, log, err => { + assert.ifError(err); + sinon.assert.calledWith( + metadata.putObjectMD.lastCall, + bucketName, + objectName, + any, + sinon.match({ overheadField: sinon.match.array }), + any, + any, + ); + done(); + }); }); }); }); it('should pass overheadField to metadata.putObjectMD for a version-suspended request', done => { - const testPutObjectRequest = versioningTestUtils - .createPutObjectRequest(bucketName, objectName, Buffer.from('I am another body', 'utf8')); + const testPutObjectRequest = versioningTestUtils.createPutObjectRequest( + bucketName, + objectName, + Buffer.from('I am another body', 'utf8'), + ); bucketPut(authInfo, testPutBucketRequest, log, () => { bucketPutVersioning(authInfo, suspendVersioningRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - err => { - assert.ifError(err); - sinon.assert.calledWith(metadata.putObjectMD.lastCall, - bucketName, objectName, any, sinon.match({ overheadField: sinon.match.array }), any, any); - done(); - } - ); + objectPut(authInfo, testPutObjectRequest, undefined, log, err => { + assert.ifError(err); + sinon.assert.calledWith( + metadata.putObjectMD.lastCall, + bucketName, + objectName, + any, + sinon.match({ overheadField: sinon.match.array }), + any, + any, + ); + done(); + }); }); }); }); it('should not pass needOplogUpdate when writing new object', done => { - async.series([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), - async () => { - sinon.assert.calledWith(metadata.putObjectMD.lastCall, - bucketName, objectName, any, sinon.match({ - needOplogUpdate: undefined, - originOp: undefined, - }), any, any); - }, - ], done); + async.series( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), + async () => { + sinon.assert.calledWith( + metadata.putObjectMD.lastCall, + bucketName, + objectName, + any, + sinon.match({ + needOplogUpdate: undefined, + originOp: undefined, + }), + any, + any, + ); + }, + ], + done, + ); }); it('should not pass needOplogUpdate when replacing object', done => { - async.series([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), - next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), - async () => { - sinon.assert.calledWith(metadata.putObjectMD.lastCall, - bucketName, objectName, any, sinon.match({ - needOplogUpdate: undefined, - originOp: undefined, - }), any, any); - }, - ], done); + async.series( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), + next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), + async () => { + sinon.assert.calledWith( + metadata.putObjectMD.lastCall, + bucketName, + objectName, + any, + sinon.match({ + needOplogUpdate: undefined, + originOp: undefined, + }), + any, + any, + ); + }, + ], + done, + ); }); it('should pass needOplogUpdate to metadata when replacing archived object', done => { const archived = { - archiveInfo: { foo: 0, bar: 'stuff' } + archiveInfo: { foo: 0, bar: 'stuff' }, }; - async.series([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), - next => fakeMetadataArchive(bucketName, objectName, undefined, archived, next), - next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), - async () => { - sinon.assert.calledWith(metadata.putObjectMD.lastCall, - bucketName, objectName, any, sinon.match({ - needOplogUpdate: true, - originOp: 's3:ReplaceArchivedObject', - }), any, any); - }, - ], done); + async.series( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), + next => fakeMetadataArchive(bucketName, objectName, undefined, archived, next), + next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), + async () => { + sinon.assert.calledWith( + metadata.putObjectMD.lastCall, + bucketName, + objectName, + any, + sinon.match({ + needOplogUpdate: true, + originOp: 's3:ReplaceArchivedObject', + }), + any, + any, + ); + }, + ], + done, + ); }); it('should pass needOplogUpdate to metadata when replacing archived object in version suspended bucket', done => { const archived = { - archiveInfo: { foo: 0, bar: 'stuff' } + archiveInfo: { foo: 0, bar: 'stuff' }, }; - async.series([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - next => bucketPutVersioning(authInfo, suspendVersioningRequest, log, next), - next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), - next => fakeMetadataArchive(bucketName, objectName, undefined, archived, next), - next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), - async () => { - sinon.assert.calledWith(metadata.putObjectMD.lastCall, - bucketName, objectName, any, sinon.match({ - needOplogUpdate: true, - originOp: 's3:ReplaceArchivedObject', - }), any, any); - }, - ], done); + async.series( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + next => bucketPutVersioning(authInfo, suspendVersioningRequest, log, next), + next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), + next => fakeMetadataArchive(bucketName, objectName, undefined, archived, next), + next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), + async () => { + sinon.assert.calledWith( + metadata.putObjectMD.lastCall, + bucketName, + objectName, + any, + sinon.match({ + needOplogUpdate: true, + originOp: 's3:ReplaceArchivedObject', + }), + any, + any, + ); + }, + ], + done, + ); }); it('should not set bucketOwnerId if requester owns the bucket', done => { - const testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: {}, - url: `/${bucketName}/${objectName}`, - calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', - }, postBody); + const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: {}, + url: `/${bucketName}/${objectName}`, + calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', + }, + postBody, + ); bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - err => { - assert.ifError(err); - sinon.assert.calledWith(metadata.putObjectMD.lastCall, - bucketName, - objectName, - sinon.match({ bucketOwnerId: sinon.match.typeOf('undefined') }), - any, - any, - any - ); - done(); - } + objectPut(authInfo, testPutObjectRequest, undefined, log, err => { + assert.ifError(err); + sinon.assert.calledWith( + metadata.putObjectMD.lastCall, + bucketName, + objectName, + sinon.match({ bucketOwnerId: sinon.match.typeOf('undefined') }), + any, + any, + any, ); + done(); + }); }); }); it('should set bucketOwnerId if requester does not own the bucket', done => { const authInfo2 = makeAuthInfo('accessKey2'); - const testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: {}, - url: `/${bucketName}/${objectName}`, - calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', - }, postBody); + const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: {}, + url: `/${bucketName}/${objectName}`, + calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', + }, + postBody, + ); const testPutPolicyRequest = new DummyRequest({ bucketName, @@ -827,42 +910,43 @@ describe('objectPut API', () => { bucketPut(authInfo, testPutBucketRequest, log, () => { bucketPutPolicy(authInfo, testPutPolicyRequest, log, err => { assert.ifError(err); - objectPut(authInfo, testPutObjectRequest, undefined, log, - err => { - assert.ifError(err); - sinon.assert.calledWith(metadata.putObjectMD.lastCall, - bucketName, - objectName, - sinon.match({ bucketOwnerId: authInfo.canonicalId }), - any, - any, - any - ); - done(); - } - ); + objectPut(authInfo, testPutObjectRequest, undefined, log, err => { + assert.ifError(err); + sinon.assert.calledWith( + metadata.putObjectMD.lastCall, + bucketName, + objectName, + sinon.match({ bucketOwnerId: authInfo.canonicalId }), + any, + any, + any, + ); + done(); + }); }); }); }); it('should fail to put object when setting a crr location as the locationConstraint', done => { - const testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { - [objectLocationConstraintHeader]: LOCATION_NAME_CRR, + const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { + [objectLocationConstraintHeader]: LOCATION_NAME_CRR, + }, + url: `/${bucketName}/${objectName}`, + calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', }, - url: `/${bucketName}/${objectName}`, - calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', - }, postBody); + postBody, + ); bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - err => { - assert(err.is.InvalidArgument); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, err => { + assert(err.is.InvalidArgument); + done(); + }); }); }); }); @@ -871,13 +955,16 @@ describe('objectPut API with versioning', () => { beforeEach(() => { cleanup(); sinon.spy(metadata, 'putObjectMD'); - testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - url: '/', - }, postBody); + testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: '/', + }, + postBody, + ); }); afterEach(() => { @@ -885,197 +972,241 @@ describe('objectPut API with versioning', () => { metadata.putObjectMD = originalputObjectMD; }); - const objData = ['foo0', 'foo1', 'foo2'].map(str => - Buffer.from(str, 'utf8')); - const testPutObjectRequests = objData.map(data => versioningTestUtils - .createPutObjectRequest(bucketName, objectName, data)); - - it('should delete latest version when creating new null version ' + - 'if latest version is null version', done => { - async.series([ - callback => bucketPut(authInfo, testPutBucketRequest, log, - callback), - // putting null version by putting obj before versioning configured - callback => objectPut(authInfo, testPutObjectRequests[0], undefined, - log, err => { - versioningTestUtils.assertDataStoreValues(ds, [objData[0]]); - callback(err); - }), - callback => bucketPutVersioning(authInfo, suspendVersioningRequest, - log, callback), - // creating new null version by putting obj after ver suspended - callback => objectPut(authInfo, testPutObjectRequests[1], - undefined, log, err => { - // wait until next tick since mem backend executes - // deletes in the next tick - setImmediate(() => { - // old null version should be deleted - versioningTestUtils.assertDataStoreValues(ds, - [undefined, objData[1]]); - callback(err); - }); - }), - // create another null version - callback => objectPut(authInfo, testPutObjectRequests[2], - undefined, log, err => { - setImmediate(() => { - // old null version should be deleted - versioningTestUtils.assertDataStoreValues(ds, - [undefined, undefined, objData[2]]); + const objData = ['foo0', 'foo1', 'foo2'].map(str => Buffer.from(str, 'utf8')); + const testPutObjectRequests = objData.map(data => + versioningTestUtils.createPutObjectRequest(bucketName, objectName, data), + ); + + it('should delete latest version when creating new null version ' + 'if latest version is null version', done => { + async.series( + [ + callback => bucketPut(authInfo, testPutBucketRequest, log, callback), + // putting null version by putting obj before versioning configured + callback => + objectPut(authInfo, testPutObjectRequests[0], undefined, log, err => { + versioningTestUtils.assertDataStoreValues(ds, [objData[0]]); callback(err); - }); - }), - ], done); + }), + callback => bucketPutVersioning(authInfo, suspendVersioningRequest, log, callback), + // creating new null version by putting obj after ver suspended + callback => + objectPut(authInfo, testPutObjectRequests[1], undefined, log, err => { + // wait until next tick since mem backend executes + // deletes in the next tick + setImmediate(() => { + // old null version should be deleted + versioningTestUtils.assertDataStoreValues(ds, [undefined, objData[1]]); + callback(err); + }); + }), + // create another null version + callback => + objectPut(authInfo, testPutObjectRequests[2], undefined, log, err => { + setImmediate(() => { + // old null version should be deleted + versioningTestUtils.assertDataStoreValues(ds, [undefined, undefined, objData[2]]); + callback(err); + }); + }), + ], + done, + ); }); describe('when null version is not the latest version', () => { - const objData = ['foo0', 'foo1', 'foo2'].map(str => - Buffer.from(str, 'utf8')); - const testPutObjectRequests = objData.map(data => versioningTestUtils - .createPutObjectRequest(bucketName, objectName, data)); + const objData = ['foo0', 'foo1', 'foo2'].map(str => Buffer.from(str, 'utf8')); + const testPutObjectRequests = objData.map(data => + versioningTestUtils.createPutObjectRequest(bucketName, objectName, data), + ); beforeEach(done => { - async.series([ - callback => bucketPut(authInfo, testPutBucketRequest, log, - callback), - // putting null version: put obj before versioning configured - callback => objectPut(authInfo, testPutObjectRequests[0], - undefined, log, callback), - callback => bucketPutVersioning(authInfo, - enableVersioningRequest, log, callback), - // put another version: - callback => objectPut(authInfo, testPutObjectRequests[1], - undefined, log, callback), - callback => bucketPutVersioning(authInfo, - suspendVersioningRequest, log, callback), - ], err => { - if (err) { - return done(err); - } - versioningTestUtils.assertDataStoreValues(ds, - objData.slice(0, 2)); - return done(); - }); + async.series( + [ + callback => bucketPut(authInfo, testPutBucketRequest, log, callback), + // putting null version: put obj before versioning configured + callback => objectPut(authInfo, testPutObjectRequests[0], undefined, log, callback), + callback => bucketPutVersioning(authInfo, enableVersioningRequest, log, callback), + // put another version: + callback => objectPut(authInfo, testPutObjectRequests[1], undefined, log, callback), + callback => bucketPutVersioning(authInfo, suspendVersioningRequest, log, callback), + ], + err => { + if (err) { + return done(err); + } + versioningTestUtils.assertDataStoreValues(ds, objData.slice(0, 2)); + return done(); + }, + ); }); - it('should still delete null version when creating new null version', - done => { - objectPut(authInfo, testPutObjectRequests[2], undefined, - log, err => { - assert.ifError(err, `Unexpected err: ${err}`); - setImmediate(() => { - // old null version should be deleted after putting - // new null version - versioningTestUtils.assertDataStoreValues(ds, - [undefined, objData[1], objData[2]]); - done(err); - }); + it('should still delete null version when creating new null version', done => { + objectPut(authInfo, testPutObjectRequests[2], undefined, log, err => { + assert.ifError(err, `Unexpected err: ${err}`); + setImmediate(() => { + // old null version should be deleted after putting + // new null version + versioningTestUtils.assertDataStoreValues(ds, [undefined, objData[1], objData[2]]); + done(err); }); + }); }); }); - it('should return BadDigest error and not leave orphans in data when ' + - 'contentMD5 and completedHash do not match', done => { - const testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: {}, - url: `/${bucketName}/${objectName}`, - contentMD5: 'vnR+tLdVF79rPPfF+7YvOg==', - }, Buffer.from('I am another body', 'utf8')); - - bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - err => { - assert.strictEqual(err.is.BadDigest, true); - // orphan objects don't get deleted - // until the next tick - // in memory - setImmediate(() => { - // Data store starts at index 1 - assert.strictEqual(ds[0], undefined); - assert.strictEqual(ds[1], undefined); - done(); + it( + 'should return BadDigest error and not leave orphans in data when ' + + 'contentMD5 and completedHash do not match', + done => { + const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: {}, + url: `/${bucketName}/${objectName}`, + contentMD5: 'vnR+tLdVF79rPPfF+7YvOg==', + }, + Buffer.from('I am another body', 'utf8'), + ); + + bucketPut(authInfo, testPutBucketRequest, log, () => { + objectPut(authInfo, testPutObjectRequest, undefined, log, err => { + assert.strictEqual(err.is.BadDigest, true); + // orphan objects don't get deleted + // until the next tick + // in memory + setImmediate(() => { + // Data store starts at index 1 + assert.strictEqual(ds[0], undefined); + assert.strictEqual(ds[1], undefined); + done(); + }); }); }); - }); - }); + }, + ); it('should set originOp when moving master-only document to a version document', done => { - async.series([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), - next => bucketPutVersioning(authInfo, enableVersioningRequest, log, next), - next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), - async () => { - // Old master-only document was moved to a proper version, with originOp overridden to prevent - // unexpected bucket notifications. - const calls = metadata.putObjectMD.getCalls(); - sinon.assert.calledWith(calls[calls.length - 2], - bucketName, objectName, sinon.match({ - originOp: 's3:StoreNullVersion', - }), any, any, any); - }, - async () => { - // New version document was created with the right originOp. - sinon.assert.calledWith(metadata.putObjectMD.lastCall, - bucketName, objectName, sinon.match({ - _data: { originOp: 's3:ObjectCreated:Put' }, - }), any, any, any); - }, - ], done); + async.series( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), + next => bucketPutVersioning(authInfo, enableVersioningRequest, log, next), + next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), + async () => { + // Old master-only document was moved to a proper version, with originOp overridden to prevent + // unexpected bucket notifications. + const calls = metadata.putObjectMD.getCalls(); + sinon.assert.calledWith( + calls[calls.length - 2], + bucketName, + objectName, + sinon.match({ + originOp: 's3:StoreNullVersion', + }), + any, + any, + any, + ); + }, + async () => { + // New version document was created with the right originOp. + sinon.assert.calledWith( + metadata.putObjectMD.lastCall, + bucketName, + objectName, + sinon.match({ + _data: { originOp: 's3:ObjectCreated:Put' }, + }), + any, + any, + any, + ); + }, + ], + done, + ); }); it('should not pass needOplogUpdate when writing new object', done => { - async.series([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - next => bucketPutVersioning(authInfo, enableVersioningRequest, log, next), - next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), - async () => { - sinon.assert.calledWith(metadata.putObjectMD.lastCall, - bucketName, objectName, any, sinon.match({ - needOplogUpdate: undefined, - originOp: undefined, - }), any, any); - }, - ], done); + async.series( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + next => bucketPutVersioning(authInfo, enableVersioningRequest, log, next), + next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), + async () => { + sinon.assert.calledWith( + metadata.putObjectMD.lastCall, + bucketName, + objectName, + any, + sinon.match({ + needOplogUpdate: undefined, + originOp: undefined, + }), + any, + any, + ); + }, + ], + done, + ); }); it('should not pass needOplogUpdate when replacing object', done => { - async.series([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - next => bucketPutVersioning(authInfo, enableVersioningRequest, log, next), - next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), - next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), - async () => { - sinon.assert.calledWith(metadata.putObjectMD.lastCall, - bucketName, objectName, any, sinon.match({ - needOplogUpdate: undefined, - originOp: undefined, - }), any, any); - }, - ], done); + async.series( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + next => bucketPutVersioning(authInfo, enableVersioningRequest, log, next), + next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), + next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), + async () => { + sinon.assert.calledWith( + metadata.putObjectMD.lastCall, + bucketName, + objectName, + any, + sinon.match({ + needOplogUpdate: undefined, + originOp: undefined, + }), + any, + any, + ); + }, + ], + done, + ); }); it('should not pass needOplogUpdate when replacing archived object', done => { const archived = { - archiveInfo: { foo: 0, bar: 'stuff' } + archiveInfo: { foo: 0, bar: 'stuff' }, }; - async.series([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - next => bucketPutVersioning(authInfo, enableVersioningRequest, log, next), - next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), - next => fakeMetadataArchive(bucketName, objectName, undefined, archived, next), - next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), - async () => { - sinon.assert.calledWith(metadata.putObjectMD.lastCall, - bucketName, objectName, any, sinon.match({ - needOplogUpdate: undefined, - originOp: undefined, - }), any, any); - }, - ], done); + async.series( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + next => bucketPutVersioning(authInfo, enableVersioningRequest, log, next), + next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), + next => fakeMetadataArchive(bucketName, objectName, undefined, archived, next), + next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), + async () => { + sinon.assert.calledWith( + metadata.putObjectMD.lastCall, + bucketName, + objectName, + any, + sinon.match({ + needOplogUpdate: undefined, + originOp: undefined, + }), + any, + any, + ); + }, + ], + done, + ); }); }); @@ -1085,10 +1216,16 @@ describe('objectPut API in ingestion bucket', () => { before(() => { // Setup multi-backend, this is required for ingestion - data.switch(new storage.data.MultipleBackendGateway({ - 'us-east-1': dataClient, - 'us-east-2': dataClient, - }, metadata, data.locStorageCheckFn)); + data.switch( + new storage.data.MultipleBackendGateway( + { + 'us-east-1': dataClient, + 'us-east-2': dataClient, + }, + metadata, + data.locStorageCheckFn, + ), + ); data.implName = 'multipleBackends'; }); @@ -1107,8 +1244,11 @@ describe('objectPut API in ingestion bucket', () => { const newPutObjectRequest = params => { const { location, versionID } = params || {}; - const r = versioningTestUtils - .createPutObjectRequest(bucketName, objectName, Buffer.from('I am another body', 'utf8')); + const r = versioningTestUtils.createPutObjectRequest( + bucketName, + objectName, + Buffer.from('I am another body', 'utf8'), + ); if (location) { r.headers[objectLocationConstraintHeader] = location; } @@ -1117,17 +1257,19 @@ describe('objectPut API in ingestion bucket', () => { } return r; }; - const newPutIngestBucketRequest = location => new DummyRequest({ - bucketName, - namespace, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - url: '/', - post: '' + - '' + - `${location}` + - '', - }); + const newPutIngestBucketRequest = location => + new DummyRequest({ + bucketName, + namespace, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: '/', + post: + '' + + '' + + `${location}` + + '', + }); const archiveRestoreRequested = { archiveInfo: { foo: 0, bar: 'stuff' }, // opaque, can be anything... restoreRequestedAt: new Date().toString(), @@ -1142,13 +1284,17 @@ describe('objectPut API in ingestion bucket', () => { cb(null, `${keyContext.bucketName}/${keyContext.objectKey}`, versionID, size, 'md5'); }); - async.series([ - next => bucketPut(authInfo, newPutIngestBucketRequest('us-east-1:ingest'), log, next), - next => objectPut(authInfo, newPutObjectRequest(), undefined, log, (err, headers) => { - assert.strictEqual(headers['x-amz-version-id'], versionID); - next(err); - }), - ], done); + async.series( + [ + next => bucketPut(authInfo, newPutIngestBucketRequest('us-east-1:ingest'), log, next), + next => + objectPut(authInfo, newPutObjectRequest(), undefined, log, (err, headers) => { + assert.strictEqual(headers['x-amz-version-id'], versionID); + next(err); + }), + ], + done, + ); }); it('should not use the versionID from the backend when writing in another location', done => { @@ -1159,16 +1305,26 @@ describe('objectPut API in ingestion bucket', () => { cb(null, `${keyContext.bucketName}/${keyContext.objectKey}`, versionID, size, 'md5'); }); - async.series([ - next => bucketPut(authInfo, newPutIngestBucketRequest('us-east-1:ingest'), log, next), - next => objectPut(authInfo, newPutObjectRequest({ - location: 'us-east-2', - }), undefined, log, (err, headers) => { - assert.ok(headers['x-amz-version-id']); - assert.notEqual(headers['x-amz-version-id'], versionID); - next(err); - }), - ], done); + async.series( + [ + next => bucketPut(authInfo, newPutIngestBucketRequest('us-east-1:ingest'), log, next), + next => + objectPut( + authInfo, + newPutObjectRequest({ + location: 'us-east-2', + }), + undefined, + log, + (err, headers) => { + assert.ok(headers['x-amz-version-id']); + assert.notEqual(headers['x-amz-version-id'], versionID); + next(err); + }, + ), + ], + done, + ); }); it('should not use the versionID from the backend when it is not a valid versionID', done => { @@ -1179,24 +1335,32 @@ describe('objectPut API in ingestion bucket', () => { cb(null, `${keyContext.bucketName}/${keyContext.objectKey}`, versionID, size, 'md5'); }); - async.series([ - next => bucketPut(authInfo, newPutIngestBucketRequest('us-east-1:ingest'), log, next), - next => objectPut(authInfo, newPutObjectRequest(), undefined, log, (err, headers) => { - assert.ok(headers['x-amz-version-id']); - assert.notEqual(headers['x-amz-version-id'], versionID); - next(err); - }), - ], done); + async.series( + [ + next => bucketPut(authInfo, newPutIngestBucketRequest('us-east-1:ingest'), log, next), + next => + objectPut(authInfo, newPutObjectRequest(), undefined, log, (err, headers) => { + assert.ok(headers['x-amz-version-id']); + assert.notEqual(headers['x-amz-version-id'], versionID); + next(err); + }), + ], + done, + ); }); it('should not use the versionID from the backend when it is not provided', done => { - async.series([ - next => bucketPut(authInfo, newPutIngestBucketRequest('us-east-1:ingest'), log, next), - next => objectPut(authInfo, newPutObjectRequest(), undefined, log, (err, headers) => { - assert.ok(headers['x-amz-version-id']); - next(err); - }), - ], done); + async.series( + [ + next => bucketPut(authInfo, newPutIngestBucketRequest('us-east-1:ingest'), log, next), + next => + objectPut(authInfo, newPutObjectRequest(), undefined, log, (err, headers) => { + assert.ok(headers['x-amz-version-id']); + next(err); + }), + ], + done, + ); }); it('should add versionID to backend putObject when restoring object', done => { @@ -1204,31 +1368,39 @@ describe('objectPut API in ingestion bucket', () => { const restoredVersionID = versioning.VersionID.encode(versioning.VersionID.generateVersionId('0', '')); // Use a "mock" data location, simulating a write to an ingest location - sinon.stub(dataClient, 'put') - .onCall(0).callsFake((writeStream, size, keyContext, reqUids, cb) => { + sinon + .stub(dataClient, 'put') + .onCall(0) + .callsFake((writeStream, size, keyContext, reqUids, cb) => { // First call: regular object creation, should not pass extra metadata header assert.strictEqual(keyContext.metaHeaders['x-amz-meta-scal-version-id'], undefined); cb(null, `${keyContext.bucketName}/${keyContext.objectKey}`, versionID, size, 'md5'); }) - .onCall(1).callsFake((writeStream, size, keyContext, reqUids, cb) => { + .onCall(1) + .callsFake((writeStream, size, keyContext, reqUids, cb) => { // Second call: "restored" data, should pass extra metadata header assert.strictEqual(keyContext.metaHeaders['x-amz-meta-scal-version-id'], versionID); cb(null, `${keyContext.bucketName}/${keyContext.objectKey}`, restoredVersionID, size, 'md5'); }); - async.series([ - next => bucketPut(authInfo, newPutIngestBucketRequest('us-east-1:ingest'), log, next), - next => objectPut(authInfo, newPutObjectRequest(), undefined, log, (err, headers) => { - assert.strictEqual(headers['x-amz-version-id'], versionID); - next(err); - }), - next => fakeMetadataArchive(bucketName, objectName, versionID, archiveRestoreRequested, next), - next => objectPut(authInfo, newPutObjectRequest({ versionID }), undefined, log, (err, headers) => { - assert.ok(headers['x-amz-version-id']); - assert.strictEqual(headers['x-amz-version-id'], versionID); // keep the same versionID - next(err); - }), - ], done); + async.series( + [ + next => bucketPut(authInfo, newPutIngestBucketRequest('us-east-1:ingest'), log, next), + next => + objectPut(authInfo, newPutObjectRequest(), undefined, log, (err, headers) => { + assert.strictEqual(headers['x-amz-version-id'], versionID); + next(err); + }), + next => fakeMetadataArchive(bucketName, objectName, versionID, archiveRestoreRequested, next), + next => + objectPut(authInfo, newPutObjectRequest({ versionID }), undefined, log, (err, headers) => { + assert.ok(headers['x-amz-version-id']); + assert.strictEqual(headers['x-amz-version-id'], versionID); // keep the same versionID + next(err); + }), + ], + done, + ); }); it('should not add versionID to backend putObject when restoring object to another location', done => { @@ -1236,34 +1408,48 @@ describe('objectPut API in ingestion bucket', () => { const restoredVersionID = versioning.VersionID.encode(versioning.VersionID.generateVersionId('0', '')); // Use a "mock" data location, simulating a write to an ingest location - sinon.stub(dataClient, 'put') - .onCall(0).callsFake((writeStream, size, keyContext, reqUids, cb) => { + sinon + .stub(dataClient, 'put') + .onCall(0) + .callsFake((writeStream, size, keyContext, reqUids, cb) => { // First call: regular object creation, should not pass extra metadata header assert.strictEqual(keyContext.metaHeaders['x-amz-meta-scal-version-id'], undefined); cb(null, `${keyContext.bucketName}/${keyContext.objectKey}`, versionID, size, 'md5'); }) - .onCall(1).callsFake((writeStream, size, keyContext, reqUids, cb) => { + .onCall(1) + .callsFake((writeStream, size, keyContext, reqUids, cb) => { // Second call: "restored" data, should not pass extra metadata header (different location) assert.strictEqual(keyContext.metaHeaders['x-amz-meta-scal-version-id'], undefined); cb(null, `${keyContext.bucketName}/${keyContext.objectKey}`, restoredVersionID, size, 'md5'); }); - async.series([ - next => bucketPut(authInfo, newPutIngestBucketRequest('us-east-1:ingest'), log, next), - next => objectPut(authInfo, newPutObjectRequest(), undefined, log, (err, headers) => { - assert.strictEqual(headers['x-amz-version-id'], versionID); - next(err); - }), - next => fakeMetadataArchive(bucketName, objectName, versionID, archiveRestoreRequested, next), - next => objectPut(authInfo, newPutObjectRequest({ - versionID, - location: 'us-east-2', - }), undefined, log, (err, headers) => { - assert.ok(headers['x-amz-version-id']); - assert.strictEqual(headers['x-amz-version-id'], versionID); // keep the same versionID - next(err); - }), - ], done); + async.series( + [ + next => bucketPut(authInfo, newPutIngestBucketRequest('us-east-1:ingest'), log, next), + next => + objectPut(authInfo, newPutObjectRequest(), undefined, log, (err, headers) => { + assert.strictEqual(headers['x-amz-version-id'], versionID); + next(err); + }), + next => fakeMetadataArchive(bucketName, objectName, versionID, archiveRestoreRequested, next), + next => + objectPut( + authInfo, + newPutObjectRequest({ + versionID, + location: 'us-east-2', + }), + undefined, + log, + (err, headers) => { + assert.ok(headers['x-amz-version-id']); + assert.strictEqual(headers['x-amz-version-id'], versionID); // keep the same versionID + next(err); + }, + ), + ], + done, + ); }); }); @@ -1278,13 +1464,17 @@ describe('objectPut with objectKeyByteLimit', () => { config.objectKeyByteLimit = originalObjectKeyByteLimit; }); - const createTestPutObjectRequest = longKey => new DummyRequest({ - bucketName, - namespace, - objectKey: longKey, - headers: {}, - url: `/${bucketName}/${longKey}`, - }, postBody); + const createTestPutObjectRequest = longKey => + new DummyRequest( + { + bucketName, + namespace, + objectKey: longKey, + headers: {}, + url: `/${bucketName}/${longKey}`, + }, + postBody, + ); it('should reject object key longer than 915 bytes by default', done => { const longKey = 'a'.repeat(916); diff --git a/tests/unit/api/objectPutACL.js b/tests/unit/api/objectPutACL.js index 5ba89d8908..61c033bf1a 100644 --- a/tests/unit/api/objectPutACL.js +++ b/tests/unit/api/objectPutACL.js @@ -6,11 +6,7 @@ const AuthInfo = require('arsenal').auth.AuthInfo; const { bucketPut } = require('../../../lib/api/bucketPut'); const bucketPutPolicy = require('../../../lib/api/bucketPutPolicy'); const constants = require('../../../constants'); -const { cleanup, - DummyRequestLogger, - makeAuthInfo, - AccessControlPolicy } - = require('../helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo, AccessControlPolicy } = require('../helpers'); const metadata = require('../metadataswitch'); const objectPut = require('../../../lib/api/objectPut'); const objectPutACL = require('../../../lib/api/objectPutACL'); @@ -20,8 +16,7 @@ const log = new DummyRequestLogger(); const canonicalID = 'accessKey1'; const authInfo = makeAuthInfo(canonicalID); const ownerID = authInfo.getCanonicalID(); -const anotherID = '79a59df900b949e55d96a1e698fba' + - 'cedfd6e09d98eacf8f8d5218e7cd47ef2bf'; +const anotherID = '79a59df900b949e55d96a1e698fba' + 'cedfd6e09d98eacf8f8d5218e7cd47ef2bf'; const defaultAcpParams = { ownerID, ownerDisplayName: 'OwnerDisplayName', @@ -42,13 +37,16 @@ let testPutObjectRequest; describe('putObjectACL API', () => { beforeEach(() => { cleanup(); - testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: {}, - url: `/${bucketName}/${objectName}`, - }, postBody); + testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: {}, + url: `/${bucketName}/${objectName}`, + }, + postBody, + ); }); it('should return an error if invalid canned ACL provided', done => { @@ -63,14 +61,13 @@ describe('putObjectACL API', () => { }; bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, resHeaders) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectPutACL(authInfo, testObjACLRequest, log, err => { - assert.strictEqual(err.is.InvalidArgument, true); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectPutACL(authInfo, testObjACLRequest, log, err => { + assert.strictEqual(err.is.InvalidArgument, true); + done(); }); + }); }); }); @@ -86,25 +83,21 @@ describe('putObjectACL API', () => { }; bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, resHeaders) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectPutACL(authInfo, testObjACLRequest, log, err => { - assert.strictEqual(err, null); - metadata.getObjectMD(bucketName, objectName, {}, - log, (err, md) => { - assert.strictEqual(md.acl.Canned, - 'public-read-write'); - assert.strictEqual(md.originOp, 's3:ObjectAcl:Put'); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectPutACL(authInfo, testObjACLRequest, log, err => { + assert.strictEqual(err, null); + metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => { + assert.strictEqual(md.acl.Canned, 'public-read-write'); + assert.strictEqual(md.originOp, 's3:ObjectAcl:Put'); + done(); }); }); + }); }); }); - it('should set a canned public-read ACL followed by' - + ' a canned authenticated-read ACL', done => { + it('should set a canned public-read ACL followed by' + ' a canned authenticated-read ACL', done => { const testObjACLRequest1 = { bucketName, namespace, @@ -126,30 +119,23 @@ describe('putObjectACL API', () => { }; bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, resHeaders) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectPutACL(authInfo, testObjACLRequest1, log, err => { - assert.strictEqual(err, null); - metadata.getObjectMD(bucketName, objectName, {}, - log, (err, md) => { - assert.strictEqual(md.acl.Canned, - 'public-read'); - objectPutACL(authInfo, testObjACLRequest2, log, - err => { - assert.strictEqual(err, null); - metadata.getObjectMD(bucketName, - objectName, {}, log, (err, md) => { - assert.strictEqual(md - .acl.Canned, - 'authenticated-read'); - assert.strictEqual(md.originOp, 's3:ObjectAcl:Put'); - done(); - }); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectPutACL(authInfo, testObjACLRequest1, log, err => { + assert.strictEqual(err, null); + metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => { + assert.strictEqual(md.acl.Canned, 'public-read'); + objectPutACL(authInfo, testObjACLRequest2, log, err => { + assert.strictEqual(err, null); + metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => { + assert.strictEqual(md.acl.Canned, 'authenticated-read'); + assert.strictEqual(md.originOp, 's3:ObjectAcl:Put'); + done(); + }); }); }); }); + }); }); }); @@ -160,8 +146,7 @@ describe('putObjectACL API', () => { objectKey: objectName, headers: { 'x-amz-grant-full-control': - 'emailaddress="sampleaccount1@sampling.com"' + - ',emailaddress="sampleaccount2@sampling.com"', + 'emailaddress="sampleaccount1@sampling.com"' + ',emailaddress="sampleaccount2@sampling.com"', 'x-amz-grant-read': `uri=${constants.logId}`, 'x-amz-grant-read-acp': `id=${ownerID}`, 'x-amz-grant-write-acp': `id=${anotherID}`, @@ -171,43 +156,34 @@ describe('putObjectACL API', () => { actionImplicitDenies: false, }; bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, resHeaders) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectPutACL(authInfo, testObjACLRequest, log, err => { + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectPutACL(authInfo, testObjACLRequest, log, err => { + assert.strictEqual(err, null); + metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => { assert.strictEqual(err, null); - metadata.getObjectMD(bucketName, objectName, {}, - log, (err, md) => { - assert.strictEqual(err, null); - const acls = md.acl; - assert.strictEqual(acls.READ[0], - constants.logId); - assert(acls.FULL_CONTROL[0] - .indexOf(ownerID) > -1); - assert(acls.FULL_CONTROL[1] - .indexOf(anotherID) > -1); - assert(acls.READ_ACP[0] - .indexOf(ownerID) > -1); - assert(acls.WRITE_ACP[0] - .indexOf(anotherID) > -1); - assert.strictEqual(md.originOp, 's3:ObjectAcl:Put'); - done(); - }); + const acls = md.acl; + assert.strictEqual(acls.READ[0], constants.logId); + assert(acls.FULL_CONTROL[0].indexOf(ownerID) > -1); + assert(acls.FULL_CONTROL[1].indexOf(anotherID) > -1); + assert(acls.READ_ACP[0].indexOf(ownerID) > -1); + assert(acls.WRITE_ACP[0].indexOf(anotherID) > -1); + assert.strictEqual(md.originOp, 's3:ObjectAcl:Put'); + done(); }); }); + }); }); }); - it('should return an error if invalid email ' + - 'provided in ACL header request', done => { + it('should return an error if invalid email ' + 'provided in ACL header request', done => { const testObjACLRequest = { bucketName, namespace, objectKey: objectName, headers: { 'x-amz-grant-full-control': - 'emailaddress="sampleaccount1@sampling.com"' + - ',emailaddress="nonexistentemail@sampling.com"', + 'emailaddress="sampleaccount1@sampling.com"' + ',emailaddress="nonexistentemail@sampling.com"', }, url: `/${bucketName}/${objectName}?acl`, query: { acl: '' }, @@ -215,24 +191,21 @@ describe('putObjectACL API', () => { }; bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, resHeaders) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectPutACL(authInfo, testObjACLRequest, log, err => { - assert.strictEqual(err.is.UnresolvableGrantByEmailAddress, true); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectPutACL(authInfo, testObjACLRequest, log, err => { + assert.strictEqual(err.is.UnresolvableGrantByEmailAddress, true); + done(); }); + }); }); }); it('should set ACLs provided in request body', done => { const acp = new AccessControlPolicy(defaultAcpParams); - acp.addGrantee('CanonicalUser', ownerID, 'FULL_CONTROL', - 'OwnerDisplayName'); + acp.addGrantee('CanonicalUser', ownerID, 'FULL_CONTROL', 'OwnerDisplayName'); acp.addGrantee('Group', constants.publicId, 'READ'); - acp.addGrantee('AmazonCustomerByEmail', 'sampleaccount1@sampling.com', - 'WRITE_ACP'); + acp.addGrantee('AmazonCustomerByEmail', 'sampleaccount1@sampling.com', 'WRITE_ACP'); acp.addGrantee('CanonicalUser', anotherID, 'READ_ACP'); const testObjACLRequest = { bucketName, @@ -246,31 +219,24 @@ describe('putObjectACL API', () => { }; bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, - log, (err, resHeaders) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectPutACL(authInfo, testObjACLRequest, log, err => { - assert.strictEqual(err, null); - metadata.getObjectMD(bucketName, objectName, {}, - log, (err, md) => { - assert.strictEqual(md - .acl.FULL_CONTROL[0], ownerID); - assert.strictEqual(md - .acl.READ[0], constants.publicId); - assert.strictEqual(md - .acl.WRITE_ACP[0], ownerID); - assert.strictEqual(md - .acl.READ_ACP[0], anotherID); - assert.strictEqual(md.originOp, 's3:ObjectAcl:Put'); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectPutACL(authInfo, testObjACLRequest, log, err => { + assert.strictEqual(err, null); + metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => { + assert.strictEqual(md.acl.FULL_CONTROL[0], ownerID); + assert.strictEqual(md.acl.READ[0], constants.publicId); + assert.strictEqual(md.acl.WRITE_ACP[0], ownerID); + assert.strictEqual(md.acl.READ_ACP[0], anotherID); + assert.strictEqual(md.originOp, 's3:ObjectAcl:Put'); + done(); }); }); + }); }); }); - it('should return an error if wrong owner ID ' + - 'provided in ACLs set out in request body', done => { + it('should return an error if wrong owner ID ' + 'provided in ACLs set out in request body', done => { const acp = new AccessControlPolicy({ ownerID: anotherID }); const testObjACLRequest = { bucketName, @@ -284,21 +250,18 @@ describe('putObjectACL API', () => { }; bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - () => { - objectPutACL(authInfo, testObjACLRequest, log, err => { - assert.strictEqual(err.is.AccessDenied, true); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, () => { + objectPutACL(authInfo, testObjACLRequest, log, err => { + assert.strictEqual(err.is.AccessDenied, true); + done(); }); + }); }); }); - it('should ignore if WRITE ACL permission is ' + - 'provided in request body', done => { + it('should ignore if WRITE ACL permission is ' + 'provided in request body', done => { const acp = new AccessControlPolicy(defaultAcpParams); - acp.addGrantee('CanonicalUser', ownerID, 'FULL_CONTROL', - 'OwnerDisplayName'); + acp.addGrantee('CanonicalUser', ownerID, 'FULL_CONTROL', 'OwnerDisplayName'); acp.addGrantee('Group', constants.publicId, 'WRITE'); const testObjACLRequest = { bucketName, @@ -312,31 +275,25 @@ describe('putObjectACL API', () => { }; bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, resHeaders) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectPutACL(authInfo, testObjACLRequest, log, err => { - assert.strictEqual(err, null); - metadata.getObjectMD(bucketName, objectName, {}, - log, (err, md) => { - assert.strictEqual(md.acl.Canned, ''); - assert.strictEqual(md.acl.FULL_CONTROL[0], - ownerID); - assert.strictEqual(md.acl.WRITE, undefined); - assert.strictEqual(md.acl.READ[0], undefined); - assert.strictEqual(md.acl.WRITE_ACP[0], - undefined); - assert.strictEqual(md.acl.READ_ACP[0], - undefined); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectPutACL(authInfo, testObjACLRequest, log, err => { + assert.strictEqual(err, null); + metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => { + assert.strictEqual(md.acl.Canned, ''); + assert.strictEqual(md.acl.FULL_CONTROL[0], ownerID); + assert.strictEqual(md.acl.WRITE, undefined); + assert.strictEqual(md.acl.READ[0], undefined); + assert.strictEqual(md.acl.WRITE_ACP[0], undefined); + assert.strictEqual(md.acl.READ_ACP[0], undefined); + done(); }); }); + }); }); }); - it('should return an error if invalid email ' + - 'address provided in ACLs set out in request body', done => { + it('should return an error if invalid email ' + 'address provided in ACLs set out in request body', done => { const acp = new AccessControlPolicy(defaultAcpParams); acp.addGrantee('AmazonCustomerByEmail', 'xyz@amazon.com', 'WRITE_ACP'); const testObjACLRequest = { @@ -350,21 +307,18 @@ describe('putObjectACL API', () => { actionImplicitDenies: false, }; - bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, resHeaders) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectPutACL(authInfo, testObjACLRequest, log, err => { - assert.strictEqual(err.is.UnresolvableGrantByEmailAddress, true); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectPutACL(authInfo, testObjACLRequest, log, err => { + assert.strictEqual(err.is.UnresolvableGrantByEmailAddress, true); + done(); }); + }); }); }); - it('should return an error if xml provided does not match s3 ' + - 'scheme for setting ACLs', done => { + it('should return an error if xml provided does not match s3 ' + 'scheme for setting ACLs', done => { const acp = new AccessControlPolicy(defaultAcpParams); acp.addGrantee('AmazonCustomerByEmail', 'xyz@amazon.com', 'WRITE_ACP'); const originalXml = acp.getXml(); @@ -381,14 +335,13 @@ describe('putObjectACL API', () => { }; bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, resHeaders) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectPutACL(authInfo, testObjACLRequest, log, err => { - assert.strictEqual(err.is.MalformedACLError, true); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectPutACL(authInfo, testObjACLRequest, log, err => { + assert.strictEqual(err.is.MalformedACLError, true); + done(); }); + }); }); }); @@ -408,24 +361,20 @@ describe('putObjectACL API', () => { actionImplicitDenies: false, }; - bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, resHeaders) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectPutACL(authInfo, testObjACLRequest, log, err => { - assert.strictEqual(err.is.MalformedXML, true); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectPutACL(authInfo, testObjACLRequest, log, err => { + assert.strictEqual(err.is.MalformedXML, true); + done(); }); + }); }); }); - it('should return an error if invalid group ' + - 'uri provided in ACLs set out in request body', done => { + it('should return an error if invalid group ' + 'uri provided in ACLs set out in request body', done => { const acp = new AccessControlPolicy(defaultAcpParams); - acp.addGrantee('Group', 'http://acs.amazonaws.com/groups/' + - 'global/NOTAVALIDGROUP', 'WRITE_ACP'); + acp.addGrantee('Group', 'http://acs.amazonaws.com/groups/' + 'global/NOTAVALIDGROUP', 'WRITE_ACP'); const testObjACLRequest = { bucketName, namespace, @@ -438,28 +387,24 @@ describe('putObjectACL API', () => { }; bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, resHeaders) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectPutACL(authInfo, testObjACLRequest, log, err => { - assert.strictEqual(err.is.InvalidArgument, true); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectPutACL(authInfo, testObjACLRequest, log, err => { + assert.strictEqual(err.is.InvalidArgument, true); + done(); }); + }); }); }); - it('should return an error if invalid group uri ' + - 'provided in ACL header request', done => { + it('should return an error if invalid group uri ' + 'provided in ACL header request', done => { const testObjACLRequest = { bucketName, namespace, objectKey: objectName, headers: { - 'host': 's3.amazonaws.com', - 'x-amz-grant-full-control': - 'uri="http://acs.amazonaws.com/groups/' + - 'global/NOTAVALIDGROUP"', + host: 's3.amazonaws.com', + 'x-amz-grant-full-control': 'uri="http://acs.amazonaws.com/groups/' + 'global/NOTAVALIDGROUP"', }, url: `/${bucketName}/${objectName}?acl`, query: { acl: '' }, @@ -467,14 +412,13 @@ describe('putObjectACL API', () => { }; bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, resHeaders) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectPutACL(authInfo, testObjACLRequest, log, err => { - assert.strictEqual(err.is.InvalidArgument, true); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectPutACL(authInfo, testObjACLRequest, log, err => { + assert.strictEqual(err.is.InvalidArgument, true); + done(); }); + }); }); }); @@ -484,56 +428,89 @@ describe('putObjectACL API', () => { { headers: { 'x-amz-grant-read': `uri=${constants.logId}` }, type: 'READ' }, { headers: { 'x-amz-grant-read-acp': `id=${ownerID}` }, type: 'READ_ACP' }, { headers: { 'x-amz-grant-write-acp': `id=${anotherID}` }, type: 'WRITE_ACP' }, - { headers: { - 'x-amz-grant-full-control': 'emailaddress="sampleaccount1@sampling.com"', - 'x-amz-grant-read': `uri=${constants.logId}`, - 'x-amz-grant-read-acp': `id=${ownerID}`, - 'x-amz-grant-write-acp': `id=${anotherID}`, - }, type: 'ALL' }, - { headers: { - 'x-amz-grant-read': `uri=${constants.logId}`, - 'x-amz-grant-read-acp': `id=${ownerID}`, - 'x-amz-grant-write-acp': `id=${anotherID}`, - }, type: 'READ/READ_ACP/WRITE_ACP' }, - { headers: { - 'x-amz-grant-full-control': 'emailaddress="sampleaccount1@sampling.com"', - 'x-amz-grant-read-acp': `id=${ownerID}`, - 'x-amz-grant-write-acp': `id=${anotherID}`, - }, type: 'FULL/READ_ACP/WRITE_ACP' }, - { headers: { - 'x-amz-grant-full-control': 'emailaddress="sampleaccount1@sampling.com"', - 'x-amz-grant-read': `uri=${constants.logId}`, - 'x-amz-grant-write-acp': `id=${anotherID}`, - }, type: 'FULL/READ/WRITE_ACP' }, - { headers: { - 'x-amz-grant-full-control': 'emailaddress="sampleaccount1@sampling.com"', - 'x-amz-grant-read': `uri=${constants.logId}`, - 'x-amz-grant-read-acp': `id=${ownerID}`, - }, type: 'FULL/READ/READ_ACP' }, - { headers: { - 'x-amz-grant-full-control': 'emailaddress="sampleaccount1@sampling.com"', - 'x-amz-grant-read': `uri=${constants.logId}`, - }, type: 'FULL/READ' }, - { headers: { - 'x-amz-grant-full-control': 'emailaddress="sampleaccount1@sampling.com"', - 'x-amz-grant-read-acp': `id=${ownerID}`, - }, type: 'FULL/READ_ACP' }, - { headers: { - 'x-amz-grant-full-control': 'emailaddress="sampleaccount1@sampling.com"', - 'x-amz-grant-write-acp': `id=${anotherID}`, - }, type: 'FULL/WRITE_ACP' }, - { headers: { - 'x-amz-grant-read': `uri=${constants.logId}`, - 'x-amz-grant-read-acp': `id=${ownerID}`, - }, type: 'READ/READ_ACP' }, - { headers: { - 'x-amz-grant-read': `uri=${constants.logId}`, - 'x-amz-grant-write-acp': `id=${anotherID}`, - }, type: 'READ/WRITE_ACP' }, - { headers: { - 'x-amz-grant-read-acp': `id=${ownerID}`, - 'x-amz-grant-write-acp': `id=${anotherID}`, - }, type: 'READ_ACP/WRITE_ACP' }, + { + headers: { + 'x-amz-grant-full-control': 'emailaddress="sampleaccount1@sampling.com"', + 'x-amz-grant-read': `uri=${constants.logId}`, + 'x-amz-grant-read-acp': `id=${ownerID}`, + 'x-amz-grant-write-acp': `id=${anotherID}`, + }, + type: 'ALL', + }, + { + headers: { + 'x-amz-grant-read': `uri=${constants.logId}`, + 'x-amz-grant-read-acp': `id=${ownerID}`, + 'x-amz-grant-write-acp': `id=${anotherID}`, + }, + type: 'READ/READ_ACP/WRITE_ACP', + }, + { + headers: { + 'x-amz-grant-full-control': 'emailaddress="sampleaccount1@sampling.com"', + 'x-amz-grant-read-acp': `id=${ownerID}`, + 'x-amz-grant-write-acp': `id=${anotherID}`, + }, + type: 'FULL/READ_ACP/WRITE_ACP', + }, + { + headers: { + 'x-amz-grant-full-control': 'emailaddress="sampleaccount1@sampling.com"', + 'x-amz-grant-read': `uri=${constants.logId}`, + 'x-amz-grant-write-acp': `id=${anotherID}`, + }, + type: 'FULL/READ/WRITE_ACP', + }, + { + headers: { + 'x-amz-grant-full-control': 'emailaddress="sampleaccount1@sampling.com"', + 'x-amz-grant-read': `uri=${constants.logId}`, + 'x-amz-grant-read-acp': `id=${ownerID}`, + }, + type: 'FULL/READ/READ_ACP', + }, + { + headers: { + 'x-amz-grant-full-control': 'emailaddress="sampleaccount1@sampling.com"', + 'x-amz-grant-read': `uri=${constants.logId}`, + }, + type: 'FULL/READ', + }, + { + headers: { + 'x-amz-grant-full-control': 'emailaddress="sampleaccount1@sampling.com"', + 'x-amz-grant-read-acp': `id=${ownerID}`, + }, + type: 'FULL/READ_ACP', + }, + { + headers: { + 'x-amz-grant-full-control': 'emailaddress="sampleaccount1@sampling.com"', + 'x-amz-grant-write-acp': `id=${anotherID}`, + }, + type: 'FULL/WRITE_ACP', + }, + { + headers: { + 'x-amz-grant-read': `uri=${constants.logId}`, + 'x-amz-grant-read-acp': `id=${ownerID}`, + }, + type: 'READ/READ_ACP', + }, + { + headers: { + 'x-amz-grant-read': `uri=${constants.logId}`, + 'x-amz-grant-write-acp': `id=${anotherID}`, + }, + type: 'READ/WRITE_ACP', + }, + { + headers: { + 'x-amz-grant-read-acp': `id=${ownerID}`, + 'x-amz-grant-write-acp': `id=${anotherID}`, + }, + type: 'READ_ACP/WRITE_ACP', + }, ].forEach(params => { const { headers, type } = params; it(`should set originOp to s3:ObjectAcl:Put when ACL is changed (${type})`, done => { @@ -546,19 +523,16 @@ describe('putObjectACL API', () => { query: { acl: '' }, }; bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, resHeaders) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectPutACL(authInfo, testObjACLRequest, log, err => { - assert.strictEqual(err, null); - metadata.getObjectMD(bucketName, objectName, {}, - log, (err, md) => { - assert.strictEqual(md.originOp, - 's3:ObjectAcl:Put'); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectPutACL(authInfo, testObjACLRequest, log, err => { + assert.strictEqual(err, null); + metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => { + assert.strictEqual(md.originOp, 's3:ObjectAcl:Put'); + done(); }); }); + }); }); }); }); @@ -574,19 +548,16 @@ describe('putObjectACL API', () => { }; bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, resHeaders) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectPutACL(authInfo, testObjACLRequest, log, err => { - assert.strictEqual(err, null); - metadata.getObjectMD(bucketName, objectName, {}, - log, (err, md) => { - assert.strictEqual(md.originOp, - 's3:ObjectCreated:Put'); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectPutACL(authInfo, testObjACLRequest, log, err => { + assert.strictEqual(err, null); + metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => { + assert.strictEqual(md.originOp, 's3:ObjectCreated:Put'); + done(); }); }); + }); }); }); @@ -608,18 +579,19 @@ describe('putObjectACL API', () => { }; beforeEach(done => { - async.waterfall([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - (cors, next) => objectPut(authInfo, - testPutObjectRequest, undefined, log, next), - ], err => { - assert.ifError(err); - done(); - }); + async.waterfall( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + (cors, next) => objectPut(authInfo, testPutObjectRequest, undefined, log, next), + ], + err => { + assert.ifError(err); + done(); + }, + ); }); - it('should succeed with a deny on unrelated object as non root', - done => { + it('should succeed with a deny on unrelated object as non root', done => { const bucketPutPolicyRequest = getPolicyRequest({ Version: '2012-10-17', Statement: [ @@ -631,36 +603,38 @@ describe('putObjectACL API', () => { }, ], }); - const testObjACLRequest = Object.assign({ - socket: { - remoteAddress: '1.1.1.1', + const testObjACLRequest = Object.assign( + { + socket: { + remoteAddress: '1.1.1.1', + }, + bucketName, + namespace, + objectKey: objectName, + headers: { 'x-amz-acl': 'public-read-write' }, + url: `/${bucketName}/${objectName}?acl`, + query: { acl: '' }, + actionImplicitDenies: false, }, - bucketName, - namespace, - objectKey: objectName, - headers: { 'x-amz-acl': 'public-read-write' }, - url: `/${bucketName}/${objectName}?acl`, - query: { acl: '' }, - actionImplicitDenies: false, - }, requestFix); + requestFix, + ); /** root user doesn't check bucket policy */ const authNotRoot = makeAuthInfo(canonicalID, 'not-root'); - async.waterfall([ - next => bucketPutPolicy(authInfo, - bucketPutPolicyRequest, log, next), - (cors, next) => objectPutACL(authNotRoot, - testObjACLRequest, log, next), - (headers, next) => metadata.getObjectMD(bucketName, - objectName, {}, log, next), - ], (err, md) => { - assert.ifError(err); - assert.strictEqual(md.acl.Canned, 'public-read-write'); - done(); - }); + async.waterfall( + [ + next => bucketPutPolicy(authInfo, bucketPutPolicyRequest, log, next), + (cors, next) => objectPutACL(authNotRoot, testObjACLRequest, log, next), + (headers, next) => metadata.getObjectMD(bucketName, objectName, {}, log, next), + ], + (err, md) => { + assert.ifError(err); + assert.strictEqual(md.acl.Canned, 'public-read-write'); + done(); + }, + ); }); - it('should fail with an allow on unrelated object as public', - done => { + it('should fail with an allow on unrelated object as public', done => { const bucketPutPolicyRequest = getPolicyRequest({ Version: '2012-10-17', Statement: [ @@ -672,31 +646,35 @@ describe('putObjectACL API', () => { }, ], }); - const testObjACLRequest = Object.assign({ - socket: { - remoteAddress: '1.1.1.1', + const testObjACLRequest = Object.assign( + { + socket: { + remoteAddress: '1.1.1.1', + }, + bucketName, + namespace, + objectKey: objectName, + headers: { 'x-amz-acl': 'public-read-write' }, + url: `/${bucketName}/${objectName}?acl`, + query: { acl: '' }, + actionImplicitDenies: false, }, - bucketName, - namespace, - objectKey: objectName, - headers: { 'x-amz-acl': 'public-read-write' }, - url: `/${bucketName}/${objectName}?acl`, - query: { acl: '' }, - actionImplicitDenies: false, - }, requestFix); + requestFix, + ); const publicAuth = new AuthInfo({ canonicalID: constants.publicId, }); - async.waterfall([ - next => bucketPutPolicy(authInfo, - bucketPutPolicyRequest, log, next), - (cors, next) => objectPutACL(publicAuth, - testObjACLRequest, log, next), - ], err => { - assert(err instanceof Error); - assert.strictEqual(err.code, errorInstances.AccessDenied.code); - done(); - }); + async.waterfall( + [ + next => bucketPutPolicy(authInfo, bucketPutPolicyRequest, log, next), + (cors, next) => objectPutACL(publicAuth, testObjACLRequest, log, next), + ], + err => { + assert(err instanceof Error); + assert.strictEqual(err.code, errorInstances.AccessDenied.code); + done(); + }, + ); }); }); }); diff --git a/tests/unit/api/objectPutLegalHold.js b/tests/unit/api/objectPutLegalHold.js index 86c41efe89..b8515ec8a6 100644 --- a/tests/unit/api/objectPutLegalHold.js +++ b/tests/unit/api/objectPutLegalHold.js @@ -22,18 +22,19 @@ const putBucketRequest = { actionImplicitDenies: false, }; -const putObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: {}, - url: `/${bucketName}/${objectName}`, -}, postBody); +const putObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: {}, + url: `/${bucketName}/${objectName}`, + }, + postBody, +); -const objectLegalHoldXml = status => '' + - `${status}` + - ''; +const objectLegalHoldXml = status => + '' + `${status}` + ''; const putLegalHoldReq = status => ({ bucketName, @@ -64,8 +65,9 @@ describe('putObjectLegalHold API', () => { }); describe('with Object Lock enabled on bucket', () => { - const bucketObjLockRequest = Object.assign({}, putBucketRequest, - { headers: { 'x-amz-bucket-object-lock-enabled': 'true' } }); + const bucketObjLockRequest = Object.assign({}, putBucketRequest, { + headers: { 'x-amz-bucket-object-lock-enabled': 'true' }, + }); beforeEach(done => { bucketPut(authInfo, bucketObjLockRequest, log, err => { @@ -75,11 +77,10 @@ describe('putObjectLegalHold API', () => { }); afterEach(cleanup); - it('should update object\'s metadata with legal hold status', done => { + it("should update object's metadata with legal hold status", done => { objectPutLegalHold(authInfo, putLegalHoldReq('ON'), log, err => { assert.ifError(err); - return metadata.getObjectMD(bucketName, objectName, {}, log, - (err, objMD) => { + return metadata.getObjectMD(bucketName, objectName, {}, log, (err, objMD) => { assert.ifError(err); assert.strictEqual(objMD.legalHold, true); return done(); @@ -87,11 +88,10 @@ describe('putObjectLegalHold API', () => { }); }); - it('should update object\'s metadata with legal hold status', done => { + it("should update object's metadata with legal hold status", done => { objectPutLegalHold(authInfo, putLegalHoldReq('OFF'), log, err => { assert.ifError(err); - return metadata.getObjectMD(bucketName, objectName, {}, log, - (err, objMD) => { + return metadata.getObjectMD(bucketName, objectName, {}, log, (err, objMD) => { assert.ifError(err); assert.strictEqual(objMD.legalHold, false); return done(); @@ -99,11 +99,10 @@ describe('putObjectLegalHold API', () => { }); }); - it('should set originOp in object\'s metadata to s3:ObjectLegalHold:Put', done => { + it("should set originOp in object's metadata to s3:ObjectLegalHold:Put", done => { objectPutLegalHold(authInfo, putLegalHoldReq('ON'), log, err => { assert.ifError(err); - return metadata.getObjectMD(bucketName, objectName, {}, log, - (err, objMD) => { + return metadata.getObjectMD(bucketName, objectName, {}, log, (err, objMD) => { assert.ifError(err); assert.strictEqual(objMD.originOp, 's3:ObjectLegalHold:Put'); return done(); diff --git a/tests/unit/api/objectPutRetention.js b/tests/unit/api/objectPutRetention.js index e0f8b19ae3..1399bca4c7 100644 --- a/tests/unit/api/objectPutRetention.js +++ b/tests/unit/api/objectPutRetention.js @@ -24,39 +24,47 @@ const bucketPutRequest = { actionImplicitDenies: false, }; -const putObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: {}, - url: `/${bucketName}/${objectName}`, -}, postBody); +const putObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: {}, + url: `/${bucketName}/${objectName}`, + }, + postBody, +); -const objectRetentionXmlGovernance = '' + 'GOVERNANCE' + `${expectedDate}` + ''; -const objectRetentionXmlCompliance = '' + 'COMPLIANCE' + `${expectedDate}` + ''; -const objectRetentionXmlGovernanceLonger = '' + 'GOVERNANCE' + `${moment().add(5, 'days').toISOString()}` + ''; -const objectRetentionXmlGovernanceShorter = '' + 'GOVERNANCE' + `${moment().add(1, 'days').toISOString()}` + ''; -const objectRetentionXmlComplianceShorter = '' + 'COMPLIANCE' + `${moment().add(1, 'days').toISOString()}` + @@ -74,7 +82,7 @@ const putObjRetRequestGovernanceWithHeader = { bucketName, objectKey: objectName, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-bypass-governance-retention': 'true', }, post: objectRetentionXmlGovernance, @@ -134,8 +142,9 @@ describe('putObjectRetention API', () => { }); describe('with Object Lock enabled on bucket', () => { - const bucketObjLockRequest = Object.assign({}, bucketPutRequest, - { headers: { 'x-amz-bucket-object-lock-enabled': 'true' } }); + const bucketObjLockRequest = Object.assign({}, bucketPutRequest, { + headers: { 'x-amz-bucket-object-lock-enabled': 'true' }, + }); beforeEach(done => { bucketPut(authInfo, bucketObjLockRequest, log, err => { @@ -145,11 +154,10 @@ describe('putObjectRetention API', () => { }); afterEach(() => cleanup()); - it('should update an object\'s metadata with retention info', done => { + it("should update an object's metadata with retention info", done => { objectPutRetention(authInfo, putObjRetRequestGovernance, log, err => { assert.ifError(err); - return metadata.getObjectMD(bucketName, objectName, {}, log, - (err, objMD) => { + return metadata.getObjectMD(bucketName, objectName, {}, log, (err, objMD) => { assert.ifError(err); assert.strictEqual(objMD.retentionMode, expectedMode); assert.strictEqual(objMD.retentionDate, expectedDate); @@ -158,11 +166,10 @@ describe('putObjectRetention API', () => { }); }); - it('should set originOp in object\'s metadata to s3:ObjectRetention:Put', done => { + it("should set originOp in object's metadata to s3:ObjectRetention:Put", done => { objectPutRetention(authInfo, putObjRetRequestGovernance, log, err => { assert.ifError(err); - return metadata.getObjectMD(bucketName, objectName, {}, log, - (err, objMD) => { + return metadata.getObjectMD(bucketName, objectName, {}, log, (err, objMD) => { assert.ifError(err); assert.strictEqual(objMD.originOp, 's3:ObjectRetention:Put'); return done(); @@ -190,48 +197,60 @@ describe('putObjectRetention API', () => { }); }); - it('should allow update if the x-amz-bypass-governance-retention header is missing and ' - + 'GOVERNANCE mode is enabled if time is being extended', done => { - objectPutRetention(authInfo, putObjRetRequestGovernance, log, err => { - assert.ifError(err); - return objectPutRetention(authInfo, putObjRetRequestGovernanceLonger, log, err => { + it( + 'should allow update if the x-amz-bypass-governance-retention header is missing and ' + + 'GOVERNANCE mode is enabled if time is being extended', + done => { + objectPutRetention(authInfo, putObjRetRequestGovernance, log, err => { assert.ifError(err); - done(); + return objectPutRetention(authInfo, putObjRetRequestGovernanceLonger, log, err => { + assert.ifError(err); + done(); + }); }); - }); - }); + }, + ); - it('should disallow update if the x-amz-bypass-governance-retention header is missing and ' - + 'GOVERNANCE mode is enabled', done => { - objectPutRetention(authInfo, putObjRetRequestGovernance, log, err => { - assert.ifError(err); - return objectPutRetention(authInfo, putObjRetRequestGovernanceShorter, log, err => { - assert.strictEqual(err.is.AccessDenied, true); - done(); + it( + 'should disallow update if the x-amz-bypass-governance-retention header is missing and ' + + 'GOVERNANCE mode is enabled', + done => { + objectPutRetention(authInfo, putObjRetRequestGovernance, log, err => { + assert.ifError(err); + return objectPutRetention(authInfo, putObjRetRequestGovernanceShorter, log, err => { + assert.strictEqual(err.is.AccessDenied, true); + done(); + }); }); - }); - }); + }, + ); - it('should allow update if the x-amz-bypass-governance-retention header is missing and ' - + 'GOVERNANCE mode is enabled and the same date is used', done => { - objectPutRetention(authInfo, putObjRetRequestGovernance, log, err => { - assert.ifError(err); - return objectPutRetention(authInfo, putObjRetRequestGovernance, log, err => { + it( + 'should allow update if the x-amz-bypass-governance-retention header is missing and ' + + 'GOVERNANCE mode is enabled and the same date is used', + done => { + objectPutRetention(authInfo, putObjRetRequestGovernance, log, err => { assert.ifError(err); - done(); + return objectPutRetention(authInfo, putObjRetRequestGovernance, log, err => { + assert.ifError(err); + done(); + }); }); - }); - }); + }, + ); - it('should allow update if the x-amz-bypass-governance-retention header is present and ' - + 'GOVERNANCE mode is enabled', done => { - objectPutRetention(authInfo, putObjRetRequestGovernance, log, err => { - assert.ifError(err); - return objectPutRetention(authInfo, putObjRetRequestGovernanceWithHeader, log, err => { + it( + 'should allow update if the x-amz-bypass-governance-retention header is present and ' + + 'GOVERNANCE mode is enabled', + done => { + objectPutRetention(authInfo, putObjRetRequestGovernance, log, err => { assert.ifError(err); - done(); + return objectPutRetention(authInfo, putObjRetRequestGovernanceWithHeader, log, err => { + assert.ifError(err); + done(); + }); }); - }); - }); + }, + ); }); }); diff --git a/tests/unit/api/objectPutTagging.js b/tests/unit/api/objectPutTagging.js index faa6d9a188..4c0a691fe0 100644 --- a/tests/unit/api/objectPutTagging.js +++ b/tests/unit/api/objectPutTagging.js @@ -3,16 +3,10 @@ const assert = require('assert'); const { bucketPut } = require('../../../lib/api/bucketPut'); const objectPut = require('../../../lib/api/objectPut'); const objectPutTagging = require('../../../lib/api/objectPutTagging'); -const { _validator, parseTagXml } - = require('arsenal').s3middleware.tagging; -const { cleanup, - DummyRequestLogger, - makeAuthInfo, - TaggingConfigTester } - = require('../helpers'); +const { _validator, parseTagXml } = require('arsenal').s3middleware.tagging; +const { cleanup, DummyRequestLogger, makeAuthInfo, TaggingConfigTester } = require('../helpers'); const metadata = require('../../../lib/metadata/wrapper'); -const { taggingTests } - = require('../../functional/aws-node-sdk/lib/utility/tagging.js'); +const { taggingTests } = require('../../functional/aws-node-sdk/lib/utility/tagging.js'); const DummyRequest = require('../DummyRequest'); const log = new DummyRequestLogger(); @@ -28,13 +22,16 @@ const testBucketPutRequest = { actionImplicitDenies: false, }; -const testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: {}, - url: `/${bucketName}/${objectName}`, -}, postBody); +const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: {}, + url: `/${bucketName}/${objectName}`, + }, + postBody, +); function _checkError(err, code, errorName) { assert(err, 'Expected error but found none'); @@ -43,14 +40,15 @@ function _checkError(err, code, errorName) { } function _generateSampleXml(key, value) { - const xml = '' + - '' + - '' + - `${key}` + - `${value}` + - '' + - '' + - ''; + const xml = + '' + + '' + + '' + + `${key}` + + `${value}` + + '' + + '' + + ''; return xml; } @@ -62,24 +60,21 @@ describe('putObjectTagging API', () => { if (err) { return done(err); } - return objectPut(authInfo, testPutObjectRequest, undefined, log, - done); + return objectPut(authInfo, testPutObjectRequest, undefined, log, done); }); }); afterEach(cleanup); - it('should update an object\'s metadata with tags resource and update originOp', done => { + it("should update an object's metadata with tags resource and update originOp", done => { const taggingUtil = new TaggingConfigTester(); - const testObjectPutTaggingRequest = taggingUtil - .createObjectTaggingRequest('PUT', bucketName, objectName); + const testObjectPutTaggingRequest = taggingUtil.createObjectTaggingRequest('PUT', bucketName, objectName); objectPutTagging(authInfo, testObjectPutTaggingRequest, log, err => { if (err) { process.stdout.write(`Err putting object tagging ${err}`); return done(err); } - return metadata.getObjectMD(bucketName, objectName, {}, log, - (err, objectMD) => { + return metadata.getObjectMD(bucketName, objectName, {}, log, (err, objectMD) => { if (err) { process.stdout.write(`Err retrieving object MD ${err}`); return done(err); @@ -95,19 +90,15 @@ describe('putObjectTagging API', () => { describe('PUT object tagging :: helper validation functions ', () => { describe('validateTagStructure ', () => { - it('should return expected true if tag is valid false/undefined if not', - done => { + it('should return expected true if tag is valid false/undefined if not', done => { const tags = [ { tagTest: { Key: ['foo'], Value: ['bar'] }, isValid: true }, { tagTest: { Key: ['foo'] }, isValid: false }, { tagTest: { Value: ['bar'] }, isValid: false }, { tagTest: { Keys: ['foo'], Value: ['bar'] }, isValid: false }, - { tagTest: { Key: ['foo', 'boo'], Value: ['bar'] }, - isValid: false }, - { tagTest: { Key: ['foo'], Value: ['bar', 'boo'] }, - isValid: false }, - { tagTest: { Key: ['foo', 'boo'], Value: ['bar', 'boo'] }, - isValid: false }, + { tagTest: { Key: ['foo', 'boo'], Value: ['bar'] }, isValid: false }, + { tagTest: { Key: ['foo'], Value: ['bar', 'boo'] }, isValid: false }, + { tagTest: { Key: ['foo', 'boo'], Value: ['bar', 'boo'] }, isValid: false }, { tagTest: { Key: ['foo'], Values: ['bar'] }, isValid: false }, { tagTest: { Keys: ['foo'], Values: ['bar'] }, isValid: false }, ]; @@ -125,26 +116,18 @@ describe('PUT object tagging :: helper validation functions ', () => { }); describe('validateXMLStructure ', () => { - it('should return expected true if tag is valid false/undefined ' + - 'if not', done => { + it('should return expected true if tag is valid false/undefined ' + 'if not', done => { const tags = [ - { tagging: { Tagging: { TagSet: [{ Tag: [] }] } }, isValid: - true }, + { tagging: { Tagging: { TagSet: [{ Tag: [] }] } }, isValid: true }, { tagging: { Tagging: { TagSet: [''] } }, isValid: true }, { tagging: { Tagging: { TagSet: [] } }, isValid: false }, { tagging: { Tagging: { TagSet: [{}] } }, isValid: false }, - { tagging: { Tagging: { Tagset: [{ Tag: [] }] } }, isValid: - false }, - { tagging: { Tagging: { Tagset: [{ Tag: [] }] }, - ExtraTagging: 'extratagging' }, isValid: false }, - { tagging: { Tagging: { Tagset: [{ Tag: [] }], ExtraTagset: - 'extratagset' } }, isValid: false }, - { tagging: { Tagging: { Tagset: [{ Tag: [] }], ExtraTagset: - 'extratagset' } }, isValid: false }, - { tagging: { Tagging: { Tagset: [{ Tag: [], ExtraTag: - 'extratag' }] } }, isValid: false }, - { tagging: { Tagging: { Tagset: [{ Tag: {} }] } }, isValid: - false }, + { tagging: { Tagging: { Tagset: [{ Tag: [] }] } }, isValid: false }, + { tagging: { Tagging: { Tagset: [{ Tag: [] }] }, ExtraTagging: 'extratagging' }, isValid: false }, + { tagging: { Tagging: { Tagset: [{ Tag: [] }], ExtraTagset: 'extratagset' } }, isValid: false }, + { tagging: { Tagging: { Tagset: [{ Tag: [] }], ExtraTagset: 'extratagset' } }, isValid: false }, + { tagging: { Tagging: { Tagset: [{ Tag: [], ExtraTag: 'extratag' }] } }, isValid: false }, + { tagging: { Tagging: { Tagset: [{ Tag: {} }] } }, isValid: false }, ]; for (let i = 0; i < tags.length; i++) { @@ -173,7 +156,9 @@ describe('PUT object tagging :: helper validation functions ', () => { taggingTests.forEach(taggingTest => { it(taggingTest.it, done => { - const { tag: { key, value } } = taggingTest; + const { + tag: { key, value }, + } = taggingTest; const xml = _generateSampleXml(key, value); parseTagXml(xml, log, (err, result) => { if (taggingTest.error) { diff --git a/tests/unit/api/objectReplicationMD.js b/tests/unit/api/objectReplicationMD.js index 48451b43ce..fa228cf3cb 100644 --- a/tests/unit/api/objectReplicationMD.js +++ b/tests/unit/api/objectReplicationMD.js @@ -4,16 +4,14 @@ const crypto = require('crypto'); const BucketInfo = require('arsenal').models.BucketInfo; -const { cleanup, DummyRequestLogger, makeAuthInfo, TaggingConfigTester } = - require('../helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo, TaggingConfigTester } = require('../helpers'); const constants = require('../../../constants'); const { metadata } = require('arsenal').storage.metadata.inMemory.metadata; const DummyRequest = require('../DummyRequest'); const { objectDelete } = require('../../../lib/api/objectDelete'); const objectPut = require('../../../lib/api/objectPut'); const objectCopy = require('../../../lib/api/objectCopy'); -const completeMultipartUpload = - require('../../../lib/api/completeMultipartUpload'); +const completeMultipartUpload = require('../../../lib/api/completeMultipartUpload'); const objectPutACL = require('../../../lib/api/objectPutACL'); const objectPutTagging = require('../../../lib/api/objectPutTagging'); const objectDeleteTagging = require('../../../lib/api/objectDeleteTagging'); @@ -55,19 +53,20 @@ const objectACLReq = { // Get an object request with the given key. function getObjectPutReq(key, hasContent) { const bodyContent = hasContent ? 'body content' : ''; - return new DummyRequest({ - bucketName, - namespace, - objectKey: key, - headers: {}, - url: `/${bucketName}/${key}`, - }, Buffer.from(bodyContent, 'utf8')); + return new DummyRequest( + { + bucketName, + namespace, + objectKey: key, + headers: {}, + url: `/${bucketName}/${key}`, + }, + Buffer.from(bodyContent, 'utf8'), + ); } -const taggingPutReq = new TaggingConfigTester() - .createObjectTaggingRequest('PUT', bucketName, keyA); -const taggingDeleteReq = new TaggingConfigTester() - .createObjectTaggingRequest('DELETE', bucketName, keyA); +const taggingPutReq = new TaggingConfigTester().createObjectTaggingRequest('PUT', bucketName, keyA); +const taggingDeleteReq = new TaggingConfigTester().createObjectTaggingRequest('DELETE', bucketName, keyA); const emptyReplicationMD = { status: '', @@ -99,35 +98,34 @@ function checkObjectReplicationInfo(key, expected) { // Put the object key and check the replication information. function putObjectAndCheckMD(key, expected, cb) { - return objectPut(authInfo, getObjectPutReq(key, true), undefined, log, - err => { - if (err) { - return cb(err); - } - checkObjectReplicationInfo(key, expected); - return cb(); - }); + return objectPut(authInfo, getObjectPutReq(key, true), undefined, log, err => { + if (err) { + return cb(err); + } + checkObjectReplicationInfo(key, expected); + return cb(); + }); } // Create the bucket in metadata. function createBucket() { - metadata - .buckets.set(bucketName, new BucketInfo(bucketName, ownerID, '', '')); - metadata.keyMaps.set(bucketName, new Map); + metadata.buckets.set(bucketName, new BucketInfo(bucketName, ownerID, '', '')); + metadata.keyMaps.set(bucketName, new Map()); } // Create the bucket in metadata with versioning and a replication config. function createBucketWithReplication(hasStorageClass) { createBucket(); const config = { - role: 'arn:aws:iam::account-id:role/src-resource,' + - 'arn:aws:iam::account-id:role/dest-resource', + role: 'arn:aws:iam::account-id:role/src-resource,' + 'arn:aws:iam::account-id:role/dest-resource', destination: 'arn:aws:s3:::source-bucket', - rules: [{ - prefix: keyA, - enabled: true, - id: 'test-id', - }], + rules: [ + { + prefix: keyA, + enabled: true, + id: 'test-id', + }, + ], }; if (hasStorageClass) { config.rules[0].storageClass = storageClassType; @@ -140,22 +138,21 @@ function createBucketWithReplication(hasStorageClass) { // Create the shadow bucket in metadata for MPUs with a recent model number. function createShadowBucket(key, uploadId) { - const overviewKey = `overview${constants.splitter}` + - `${key}${constants.splitter}${uploadId}`; - metadata.buckets - .set(mpuShadowBucket, new BucketInfo(mpuShadowBucket, ownerID, '', '')); - // Set modelVersion to use the most recent splitter. + const overviewKey = `overview${constants.splitter}` + `${key}${constants.splitter}${uploadId}`; + metadata.buckets.set(mpuShadowBucket, new BucketInfo(mpuShadowBucket, ownerID, '', '')); + // Set modelVersion to use the most recent splitter. Object.assign(metadata.buckets.get(mpuShadowBucket), { _mdBucketModelVersion: 5, }); - metadata.keyMaps.set(mpuShadowBucket, new Map); - metadata.keyMaps.get(mpuShadowBucket).set(overviewKey, new Map); + metadata.keyMaps.set(mpuShadowBucket, new Map()); + metadata.keyMaps.get(mpuShadowBucket).set(overviewKey, new Map()); Object.assign(metadata.keyMaps.get(mpuShadowBucket).get(overviewKey), { id: uploadId, eventualStorageBucket: bucketName, initiator: { DisplayName: 'accessKey1displayName', - ID: ownerID }, + ID: ownerID, + }, key, uploadId, }); @@ -170,24 +167,26 @@ function putMPU(key, body, cb) { const calculatedHash = md5Hash.digest('hex'); const partKey = `${uploadId}${constants.splitter}00001`; const obj = { - partLocations: [{ - key: 1, - dataStoreName: 'scality-internal-mem', - dataStoreETag: `1:${calculatedHash}`, - }], + partLocations: [ + { + key: 1, + dataStoreName: 'scality-internal-mem', + dataStoreETag: `1:${calculatedHash}`, + }, + ], key: partKey, }; obj['content-md5'] = calculatedHash; obj['content-length'] = body.length; - metadata.keyMaps.get(mpuShadowBucket).set(partKey, new Map); + metadata.keyMaps.get(mpuShadowBucket).set(partKey, new Map()); const partMap = metadata.keyMaps.get(mpuShadowBucket).get(partKey); Object.assign(partMap, obj); const postBody = '' + - '' + - '1' + - `"${calculatedHash}"` + - '' + + '' + + '1' + + `"${calculatedHash}"` + + '' + ''; const req = { bucketName, @@ -217,8 +216,7 @@ function copyObject(sourceObjectKey, copyObjectKey, hasContent, cb) { headers: {}, url: `/${bucketName}/${sourceObjectKey}`, }); - return objectCopy(authInfo, req, bucketName, sourceObjectKey, undefined, - log, cb); + return objectCopy(authInfo, req, bucketName, sourceObjectKey, undefined, log, cb); }); } @@ -230,26 +228,33 @@ describe('Replication object MD without bucket replication config', () => { afterEach(() => cleanup()); - it('should not update object metadata', done => - putObjectAndCheckMD(keyA, emptyReplicationMD, done)); + it('should not update object metadata', done => putObjectAndCheckMD(keyA, emptyReplicationMD, done)); it('should not update object metadata if putting object ACL', done => - async.series([ - next => putObjectAndCheckMD(keyA, emptyReplicationMD, next), - next => objectPutACL(authInfo, objectACLReq, log, next), - ], err => { - if (err) { - return done(err); - } - checkObjectReplicationInfo(keyA, expectedEmptyReplicationMD); - return done(); - })); + async.series( + [ + next => putObjectAndCheckMD(keyA, emptyReplicationMD, next), + next => objectPutACL(authInfo, objectACLReq, log, next), + ], + err => { + if (err) { + return done(err); + } + checkObjectReplicationInfo(keyA, expectedEmptyReplicationMD); + return done(); + }, + )); describe('Object tagging', () => { - beforeEach(done => async.series([ - next => putObjectAndCheckMD(keyA, emptyReplicationMD, next), - next => objectPutTagging(authInfo, taggingPutReq, log, next), - ], err => done(err))); + beforeEach(done => + async.series( + [ + next => putObjectAndCheckMD(keyA, emptyReplicationMD, next), + next => objectPutTagging(authInfo, taggingPutReq, log, next), + ], + err => done(err), + ), + ); it('should not update object metadata if putting tag', done => { checkObjectReplicationInfo(keyA, expectedEmptyReplicationMD); @@ -257,18 +262,20 @@ describe('Replication object MD without bucket replication config', () => { }); it('should not update object metadata if deleting tag', done => - async.series([ - // Put a new version to update replication MD content array. - next => putObjectAndCheckMD(keyA, emptyReplicationMD, next), - next => objectDeleteTagging(authInfo, taggingDeleteReq, log, - next), - ], err => { - if (err) { - return done(err); - } - checkObjectReplicationInfo(keyA, expectedEmptyReplicationMD); - return done(); - })); + async.series( + [ + // Put a new version to update replication MD content array. + next => putObjectAndCheckMD(keyA, emptyReplicationMD, next), + next => objectDeleteTagging(authInfo, taggingDeleteReq, log, next), + ], + err => { + if (err) { + return done(err); + } + checkObjectReplicationInfo(keyA, expectedEmptyReplicationMD); + return done(); + }, + )); it('should not update object metadata if completing MPU', done => putMPU(keyA, 'content', err => { @@ -291,430 +298,455 @@ describe('Replication object MD without bucket replication config', () => { }); [true, false].forEach(hasStorageClass => { - describe('Replication object MD with bucket replication config ' + - `${hasStorageClass ? 'with' : 'without'} storage class`, () => { - const replicationMD = { - status: 'PENDING', - backends: [{ - site: 'zenko', + describe( + 'Replication object MD with bucket replication config ' + + `${hasStorageClass ? 'with' : 'without'} storage class`, + () => { + const replicationMD = { status: 'PENDING', + backends: [ + { + site: 'zenko', + status: 'PENDING', + dataStoreVersionId: '', + }, + ], + content: ['DATA', 'METADATA'], + destination: bucketARN, + storageClass: 'zenko', + role: 'arn:aws:iam::account-id:role/src-resource,' + 'arn:aws:iam::account-id:role/dest-resource', + storageType: '', dataStoreVersionId: '', - }], - content: ['DATA', 'METADATA'], - destination: bucketARN, - storageClass: 'zenko', - role: 'arn:aws:iam::account-id:role/src-resource,' + - 'arn:aws:iam::account-id:role/dest-resource', - storageType: '', - dataStoreVersionId: '', - isNFS: undefined, - }; - const newReplicationMD = hasStorageClass ? Object.assign(replicationMD, - { storageClass: storageClassType }) : replicationMD; - const replicateMetadataOnly = Object.assign({}, newReplicationMD, - { content: ['METADATA'] }); - - beforeEach(() => { - cleanup(); - createBucketWithReplication(hasStorageClass); - }); - - afterEach(() => { - cleanup(); - delete config.locationConstraints['zenko']; - }); - - it('should update metadata when replication config prefix matches ' + - 'an object key', done => - putObjectAndCheckMD(keyA, newReplicationMD, done)); - - it('should update metadata when replication config prefix matches ' + - 'the start of an object key', done => - putObjectAndCheckMD(`${keyA}abc`, newReplicationMD, done)); - - it('should not update metadata when replication config prefix does ' + - 'not match the start of an object key', done => - putObjectAndCheckMD(`abc${keyA}`, emptyReplicationMD, done)); - - it('should not update metadata when replication config prefix does ' + - 'not apply', done => - putObjectAndCheckMD(keyB, emptyReplicationMD, done)); - - it("should update status to 'PENDING' if putting a new version", done => - putObjectAndCheckMD(keyA, newReplicationMD, err => { - if (err) { - return done(err); - } - const objectMD = metadata.keyMaps.get(bucketName).get(keyA); - // Update metadata to a status after replication has occurred. - objectMD.replicationInfo.status = 'COMPLETED'; - return putObjectAndCheckMD(keyA, newReplicationMD, done); - })); - - it("should update status to 'PENDING' and content to '['METADATA']' " + - 'if putting 0 byte object', done => - objectPut(authInfo, getObjectPutReq(keyA, false), undefined, log, - err => { - if (err) { - return done(err); - } - checkObjectReplicationInfo(keyA, replicateMetadataOnly); - return done(); - })); - - it('should update metadata if putting object ACL and CRR replication', done => { - // Set 'zenko' as a typical CRR location (i.e. no type) - config.locationConstraints['zenko'] = { - ...config.locationConstraints['zenko'], - type: '', + isNFS: undefined, }; - - async.series([ - next => putObjectAndCheckMD(keyA, newReplicationMD, next), - next => { - const objectMD = metadata.keyMaps.get(bucketName).get(keyA); - // Update metadata to a status after replication has occurred. - objectMD.replicationInfo.status = 'COMPLETED'; - objectPutACL(authInfo, objectACLReq, log, next); - }, - ], err => { - if (err) { - return done(err); - } - checkObjectReplicationInfo(keyA, replicateMetadataOnly); - return done(); + const newReplicationMD = hasStorageClass + ? Object.assign(replicationMD, { storageClass: storageClassType }) + : replicationMD; + const replicateMetadataOnly = Object.assign({}, newReplicationMD, { content: ['METADATA'] }); + + beforeEach(() => { + cleanup(); + createBucketWithReplication(hasStorageClass); }); - }); - - it('should not update metadata if putting object ACL and cloud replication', done => { - // Set 'zenko' as a typical cloud location (i.e. type) - config.locationConstraints['zenko'] = { - ...config.locationConstraints['zenko'], - type: 'aws_s3', - }; - const replicationMD = { ...newReplicationMD, storageType: 'aws_s3' }; - - let completedReplicationInfo; - async.series([ - next => putObjectAndCheckMD(keyA, replicationMD, next), - next => { - const objectMD = metadata.keyMaps.get(bucketName).get(keyA); - // Update metadata to a status after replication has occurred. - objectMD.replicationInfo.status = 'COMPLETED'; - completedReplicationInfo = JSON.parse( - JSON.stringify(objectMD.replicationInfo)); - objectPutACL(authInfo, objectACLReq, log, next); - }, - ], err => { - if (err) { - return done(err); - } - checkObjectReplicationInfo(keyA, completedReplicationInfo); - return done(); + afterEach(() => { + cleanup(); + delete config.locationConstraints['zenko']; }); - }); - - it('should update metadata if putting a delete marker', done => - async.series([ - next => putObjectAndCheckMD(keyA, newReplicationMD, err => { - if (err) { - return next(err); - } - const objectMD = metadata.keyMaps.get(bucketName).get(keyA); - // Set metadata to a status after replication has occurred. - objectMD.replicationInfo.status = 'COMPLETED'; - return next(); - }), - next => objectDelete(authInfo, deleteReq, log, next), - ], err => { - if (err) { - return done(err); - } - const objectMD = metadata.keyMaps.get(bucketName).get(keyA); - assert.strictEqual(objectMD.isDeleteMarker, true); - checkObjectReplicationInfo(keyA, replicateMetadataOnly); - return done(); - })); - it('should not update metadata if putting a delete marker owned by ' + - 'Lifecycle service account', done => - async.series([ - next => putObjectAndCheckMD(keyA, newReplicationMD, next), - next => objectDelete(authInfoLifecycleService, deleteReq, - log, next), - ], err => { - if (err) { - return done(err); - } - const objectMD = metadata.keyMaps.get(bucketName).get(keyA); - assert.strictEqual(objectMD.isDeleteMarker, true); - checkObjectReplicationInfo(keyA, emptyReplicationMD); - return done(); - })); + it('should update metadata when replication config prefix matches ' + 'an object key', done => + putObjectAndCheckMD(keyA, newReplicationMD, done), + ); - describe('Object tagging', () => { - beforeEach(done => async.series([ - next => putObjectAndCheckMD(keyA, newReplicationMD, next), - next => objectPutTagging(authInfo, taggingPutReq, log, next), - ], err => done(err))); + it('should update metadata when replication config prefix matches ' + 'the start of an object key', done => + putObjectAndCheckMD(`${keyA}abc`, newReplicationMD, done), + ); - it("should update status to 'PENDING' and content to " + - "'['METADATA']'if putting tag", done => { - checkObjectReplicationInfo(keyA, replicateMetadataOnly); - return done(); - }); + it( + 'should not update metadata when replication config prefix does ' + + 'not match the start of an object key', + done => putObjectAndCheckMD(`abc${keyA}`, emptyReplicationMD, done), + ); - it("should update status to 'PENDING' and content to " + - "'['METADATA']' if deleting tag", done => - async.series([ - // Put a new version to update replication MD content array. - next => putObjectAndCheckMD(keyA, newReplicationMD, next), - next => objectDeleteTagging(authInfo, taggingDeleteReq, log, - next), - ], err => { - if (err) { - return done(err); - } - checkObjectReplicationInfo(keyA, replicateMetadataOnly); - return done(); - })); - }); + it('should not update metadata when replication config prefix does ' + 'not apply', done => + putObjectAndCheckMD(keyB, emptyReplicationMD, done), + ); - describe('Complete MPU', () => { - it("should update status to 'PENDING' and content to " + - "'['DATA, METADATA']' if completing MPU", done => - putMPU(keyA, 'content', err => { + it("should update status to 'PENDING' if putting a new version", done => + putObjectAndCheckMD(keyA, newReplicationMD, err => { if (err) { return done(err); } - checkObjectReplicationInfo(keyA, newReplicationMD); - return done(); + const objectMD = metadata.keyMaps.get(bucketName).get(keyA); + // Update metadata to a status after replication has occurred. + objectMD.replicationInfo.status = 'COMPLETED'; + return putObjectAndCheckMD(keyA, newReplicationMD, done); })); - it("should update status to 'PENDING' and content to " + - "'['METADATA']' if completing MPU with 0 bytes", done => - putMPU(keyA, '', err => { + it("should update status to 'PENDING' and content to '['METADATA']' " + 'if putting 0 byte object', done => + objectPut(authInfo, getObjectPutReq(keyA, false), undefined, log, err => { if (err) { return done(err); } checkObjectReplicationInfo(keyA, replicateMetadataOnly); return done(); - })); - - it('should not update replicationInfo if key does not apply', - done => putMPU(keyB, 'content', err => { - if (err) { - return done(err); - } - checkObjectReplicationInfo(keyB, emptyReplicationMD); - return done(); - })); - }); - - describe('Object copy', () => { - it("should update status to 'PENDING' and content to " + - "'['DATA, METADATA']' if copying object", done => - copyObject(keyB, keyA, true, err => { - if (err) { - return done(err); - } - checkObjectReplicationInfo(keyA, newReplicationMD); - return done(); - })); + }), + ); - it("should update status to 'PENDING' and content to " + - "'['METADATA']' if copying object with 0 bytes", done => - copyObject(keyB, keyA, false, err => { - if (err) { - return done(err); - } - checkObjectReplicationInfo(keyA, replicateMetadataOnly); - return done(); - })); + it('should update metadata if putting object ACL and CRR replication', done => { + // Set 'zenko' as a typical CRR location (i.e. no type) + config.locationConstraints['zenko'] = { + ...config.locationConstraints['zenko'], + type: '', + }; - it('should not update replicationInfo if key does not apply', - done => { - const copyKey = `foo-${keyA}`; - return copyObject(keyB, copyKey, true, err => { + async.series( + [ + next => putObjectAndCheckMD(keyA, newReplicationMD, next), + next => { + const objectMD = metadata.keyMaps.get(bucketName).get(keyA); + // Update metadata to a status after replication has occurred. + objectMD.replicationInfo.status = 'COMPLETED'; + objectPutACL(authInfo, objectACLReq, log, next); + }, + ], + err => { if (err) { return done(err); } - checkObjectReplicationInfo(copyKey, emptyReplicationMD); + checkObjectReplicationInfo(keyA, replicateMetadataOnly); return done(); - }); - }); - }); + }, + ); + }); - ['awsbackend', - 'azurebackend', - 'gcpbackend', - 'awsbackend,azurebackend'].forEach(backend => { - const storageTypeMap = { - 'awsbackend': 'aws_s3', - 'azurebackend': 'azure', - 'gcpbackend': 'gcp', - 'awsbackend,azurebackend': 'aws_s3,azure', - }; - const storageType = storageTypeMap[backend]; - const backends = backend.split(',').map(site => ({ - site, - status: 'PENDING', - dataStoreVersionId: '', - })); - describe('Object metadata replicationInfo storageType value', - () => { - const expectedReplicationInfo = { - status: 'PENDING', - backends, - content: ['DATA', 'METADATA'], - destination: 'arn:aws:s3:::destination-bucket', - storageClass: backend, - role: 'arn:aws:iam::account-id:role/resource', - storageType, - dataStoreVersionId: '', - isNFS: undefined, + it('should not update metadata if putting object ACL and cloud replication', done => { + // Set 'zenko' as a typical cloud location (i.e. type) + config.locationConstraints['zenko'] = { + ...config.locationConstraints['zenko'], + type: 'aws_s3', }; - // Expected for a metadata-only replication operation (for - // example, putting object tags). - const expectedReplicationInfoMD = Object.assign({}, - expectedReplicationInfo, { content: ['METADATA'] }); - - beforeEach(() => - // We have already created the bucket, so update the - // replication configuration to include a location - // constraint for the `storageClass`. This results in a - // `storageType` of 'aws_s3', for example. - Object.assign(metadata.buckets.get(bucketName), { - _replicationConfiguration: { - role: 'arn:aws:iam::account-id:role/resource', - destination: 'arn:aws:s3:::destination-bucket', - rules: [{ - prefix: keyA, - enabled: true, - id: 'test-id', - storageClass: backend, - }], - }, - })); + const replicationMD = { ...newReplicationMD, storageType: 'aws_s3' }; - it('should update on a put object request', done => - putObjectAndCheckMD(keyA, expectedReplicationInfo, done)); - - it('should update on a complete MPU object request', done => - putMPU(keyA, 'content', err => { + let completedReplicationInfo; + async.series( + [ + next => putObjectAndCheckMD(keyA, replicationMD, next), + next => { + const objectMD = metadata.keyMaps.get(bucketName).get(keyA); + // Update metadata to a status after replication has occurred. + objectMD.replicationInfo.status = 'COMPLETED'; + completedReplicationInfo = JSON.parse(JSON.stringify(objectMD.replicationInfo)); + objectPutACL(authInfo, objectACLReq, log, next); + }, + ], + err => { if (err) { return done(err); } - const expected = - Object.assign({}, expectedReplicationInfo, - { content: ['DATA', 'METADATA', 'MPU'] }); - checkObjectReplicationInfo(keyA, expected); + checkObjectReplicationInfo(keyA, completedReplicationInfo); return done(); - })); + }, + ); + }); - it('should update on a copy object request', done => - copyObject(keyB, keyA, true, err => { + it('should update metadata if putting a delete marker', done => + async.series( + [ + next => + putObjectAndCheckMD(keyA, newReplicationMD, err => { + if (err) { + return next(err); + } + const objectMD = metadata.keyMaps.get(bucketName).get(keyA); + // Set metadata to a status after replication has occurred. + objectMD.replicationInfo.status = 'COMPLETED'; + return next(); + }), + next => objectDelete(authInfo, deleteReq, log, next), + ], + err => { if (err) { return done(err); } - checkObjectReplicationInfo(keyA, - expectedReplicationInfo); + const objectMD = metadata.keyMaps.get(bucketName).get(keyA); + assert.strictEqual(objectMD.isDeleteMarker, true); + checkObjectReplicationInfo(keyA, replicateMetadataOnly); return done(); - })); - - it('should update on a put object ACL request', done => { - let completedReplicationInfo; - async.series([ - next => putObjectAndCheckMD(keyA, - expectedReplicationInfo, next), - next => { - const objectMD = metadata.keyMaps - .get(bucketName).get(keyA); - // Update metadata to a status after replication - // has occurred. - objectMD.replicationInfo.status = 'COMPLETED'; - completedReplicationInfo = JSON.parse( - JSON.stringify(objectMD.replicationInfo)); - objectPutACL(authInfo, objectACLReq, log, next); - }, - ], err => { + }, + )); + + it('should not update metadata if putting a delete marker owned by ' + 'Lifecycle service account', done => + async.series( + [ + next => putObjectAndCheckMD(keyA, newReplicationMD, next), + next => objectDelete(authInfoLifecycleService, deleteReq, log, next), + ], + err => { if (err) { return done(err); } - checkObjectReplicationInfo(keyA, completedReplicationInfo); + const objectMD = metadata.keyMaps.get(bucketName).get(keyA); + assert.strictEqual(objectMD.isDeleteMarker, true); + checkObjectReplicationInfo(keyA, emptyReplicationMD); return done(); - }); + }, + ), + ); + + describe('Object tagging', () => { + beforeEach(done => + async.series( + [ + next => putObjectAndCheckMD(keyA, newReplicationMD, next), + next => objectPutTagging(authInfo, taggingPutReq, log, next), + ], + err => done(err), + ), + ); + + it("should update status to 'PENDING' and content to " + "'['METADATA']'if putting tag", done => { + checkObjectReplicationInfo(keyA, replicateMetadataOnly); + return done(); }); - it('should update on a put object tagging request', done => - async.series([ - next => putObjectAndCheckMD(keyA, - expectedReplicationInfo, next), - next => objectPutTagging(authInfo, taggingPutReq, log, - next), - ], err => { + it("should update status to 'PENDING' and content to " + "'['METADATA']' if deleting tag", done => + async.series( + [ + // Put a new version to update replication MD content array. + next => putObjectAndCheckMD(keyA, newReplicationMD, next), + next => objectDeleteTagging(authInfo, taggingDeleteReq, log, next), + ], + err => { + if (err) { + return done(err); + } + checkObjectReplicationInfo(keyA, replicateMetadataOnly); + return done(); + }, + ), + ); + }); + + describe('Complete MPU', () => { + it( + "should update status to 'PENDING' and content to " + "'['DATA, METADATA']' if completing MPU", + done => + putMPU(keyA, 'content', err => { + if (err) { + return done(err); + } + checkObjectReplicationInfo(keyA, newReplicationMD); + return done(); + }), + ); + + it( + "should update status to 'PENDING' and content to " + + "'['METADATA']' if completing MPU with 0 bytes", + done => + putMPU(keyA, '', err => { + if (err) { + return done(err); + } + checkObjectReplicationInfo(keyA, replicateMetadataOnly); + return done(); + }), + ); + + it('should not update replicationInfo if key does not apply', done => + putMPU(keyB, 'content', err => { if (err) { return done(err); } - const expected = Object.assign({}, - expectedReplicationInfo, - { content: ['METADATA', 'PUT_TAGGING'] }); - checkObjectReplicationInfo(keyA, expected); + checkObjectReplicationInfo(keyB, emptyReplicationMD); return done(); })); + }); - it('should update on a delete tagging request', done => - async.series([ - next => putObjectAndCheckMD(keyA, - expectedReplicationInfo, next), - next => objectDeleteTagging(authInfo, taggingDeleteReq, - log, next), - ], err => { + describe('Object copy', () => { + it( + "should update status to 'PENDING' and content to " + "'['DATA, METADATA']' if copying object", + done => + copyObject(keyB, keyA, true, err => { + if (err) { + return done(err); + } + checkObjectReplicationInfo(keyA, newReplicationMD); + return done(); + }), + ); + + it( + "should update status to 'PENDING' and content to " + + "'['METADATA']' if copying object with 0 bytes", + done => + copyObject(keyB, keyA, false, err => { + if (err) { + return done(err); + } + checkObjectReplicationInfo(keyA, replicateMetadataOnly); + return done(); + }), + ); + + it('should not update replicationInfo if key does not apply', done => { + const copyKey = `foo-${keyA}`; + return copyObject(keyB, copyKey, true, err => { if (err) { return done(err); } - const expected = Object.assign({}, - expectedReplicationInfo, - { content: ['METADATA', 'DELETE_TAGGING'] }); - checkObjectReplicationInfo(keyA, expected); + checkObjectReplicationInfo(copyKey, emptyReplicationMD); return done(); - })); + }); + }); + }); - it('should update when putting a delete marker', done => - async.series([ - next => putObjectAndCheckMD(keyA, - expectedReplicationInfo, err => { + ['awsbackend', 'azurebackend', 'gcpbackend', 'awsbackend,azurebackend'].forEach(backend => { + const storageTypeMap = { + awsbackend: 'aws_s3', + azurebackend: 'azure', + gcpbackend: 'gcp', + 'awsbackend,azurebackend': 'aws_s3,azure', + }; + const storageType = storageTypeMap[backend]; + const backends = backend.split(',').map(site => ({ + site, + status: 'PENDING', + dataStoreVersionId: '', + })); + describe('Object metadata replicationInfo storageType value', () => { + const expectedReplicationInfo = { + status: 'PENDING', + backends, + content: ['DATA', 'METADATA'], + destination: 'arn:aws:s3:::destination-bucket', + storageClass: backend, + role: 'arn:aws:iam::account-id:role/resource', + storageType, + dataStoreVersionId: '', + isNFS: undefined, + }; + + // Expected for a metadata-only replication operation (for + // example, putting object tags). + const expectedReplicationInfoMD = Object.assign({}, expectedReplicationInfo, { + content: ['METADATA'], + }); + + beforeEach(() => + // We have already created the bucket, so update the + // replication configuration to include a location + // constraint for the `storageClass`. This results in a + // `storageType` of 'aws_s3', for example. + Object.assign(metadata.buckets.get(bucketName), { + _replicationConfiguration: { + role: 'arn:aws:iam::account-id:role/resource', + destination: 'arn:aws:s3:::destination-bucket', + rules: [ + { + prefix: keyA, + enabled: true, + id: 'test-id', + storageClass: backend, + }, + ], + }, + }), + ); + + it('should update on a put object request', done => + putObjectAndCheckMD(keyA, expectedReplicationInfo, done)); + + it('should update on a complete MPU object request', done => + putMPU(keyA, 'content', err => { + if (err) { + return done(err); + } + const expected = Object.assign({}, expectedReplicationInfo, { + content: ['DATA', 'METADATA', 'MPU'], + }); + checkObjectReplicationInfo(keyA, expected); + return done(); + })); + + it('should update on a copy object request', done => + copyObject(keyB, keyA, true, err => { + if (err) { + return done(err); + } + checkObjectReplicationInfo(keyA, expectedReplicationInfo); + return done(); + })); + + it('should update on a put object ACL request', done => { + let completedReplicationInfo; + async.series( + [ + next => putObjectAndCheckMD(keyA, expectedReplicationInfo, next), + next => { + const objectMD = metadata.keyMaps.get(bucketName).get(keyA); + // Update metadata to a status after replication + // has occurred. + objectMD.replicationInfo.status = 'COMPLETED'; + completedReplicationInfo = JSON.parse(JSON.stringify(objectMD.replicationInfo)); + objectPutACL(authInfo, objectACLReq, log, next); + }, + ], + err => { if (err) { - return next(err); + return done(err); } - // Update metadata to a status indicating that - // replication has occurred for the object. - metadata - .keyMaps - .get(bucketName) - .get(keyA) - .replicationInfo - .status = 'COMPLETED'; - return next(); - }), - next => objectDelete(authInfo, deleteReq, log, next), - ], err => { - if (err) { - return done(err); - } - // Is it, in fact, a delete marker? - assert(metadata - .keyMaps - .get(bucketName) - .get(keyA) - .isDeleteMarker); - checkObjectReplicationInfo(keyA, - expectedReplicationInfoMD); - return done(); - })); + checkObjectReplicationInfo(keyA, completedReplicationInfo); + return done(); + }, + ); + }); + + it('should update on a put object tagging request', done => + async.series( + [ + next => putObjectAndCheckMD(keyA, expectedReplicationInfo, next), + next => objectPutTagging(authInfo, taggingPutReq, log, next), + ], + err => { + if (err) { + return done(err); + } + const expected = Object.assign({}, expectedReplicationInfo, { + content: ['METADATA', 'PUT_TAGGING'], + }); + checkObjectReplicationInfo(keyA, expected); + return done(); + }, + )); + + it('should update on a delete tagging request', done => + async.series( + [ + next => putObjectAndCheckMD(keyA, expectedReplicationInfo, next), + next => objectDeleteTagging(authInfo, taggingDeleteReq, log, next), + ], + err => { + if (err) { + return done(err); + } + const expected = Object.assign({}, expectedReplicationInfo, { + content: ['METADATA', 'DELETE_TAGGING'], + }); + checkObjectReplicationInfo(keyA, expected); + return done(); + }, + )); + + it('should update when putting a delete marker', done => + async.series( + [ + next => + putObjectAndCheckMD(keyA, expectedReplicationInfo, err => { + if (err) { + return next(err); + } + // Update metadata to a status indicating that + // replication has occurred for the object. + metadata.keyMaps.get(bucketName).get(keyA).replicationInfo.status = 'COMPLETED'; + return next(); + }), + next => objectDelete(authInfo, deleteReq, log, next), + ], + err => { + if (err) { + return done(err); + } + // Is it, in fact, a delete marker? + assert(metadata.keyMaps.get(bucketName).get(keyA).isDeleteMarker); + checkObjectReplicationInfo(keyA, expectedReplicationInfoMD); + return done(); + }, + )); + }); }); - }); - }); + }, + ); }); diff --git a/tests/unit/api/objectRestore.js b/tests/unit/api/objectRestore.js index 28b16248ef..67e3b6a560 100644 --- a/tests/unit/api/objectRestore.js +++ b/tests/unit/api/objectRestore.js @@ -25,38 +25,44 @@ const bucketPutRequest = { actionImplicitDenies: false, }; -const putObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: {}, - url: `/${bucketName}/${objectName}`, -}, postBody); - -const objectRestoreXml = '' + `${restoreDays}` + 'Standard' + ''; -const objectRestoreXmlBulkTier = '' + `${restoreDays}` + 'Bulk' + ''; -const objectRestoreXmlExpeditedTier = '' + `${restoreDays}` + 'Expedited' + ''; const objectRestoreRequest = requestXml => ({ - bucketName, - objectKey: objectName, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - post: requestXml, - }); + bucketName, + objectKey: objectName, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + post: requestXml, +}); describe('restoreObject API', () => { before(cleanup); @@ -87,7 +93,7 @@ describe('restoreObject API', () => { }); }); - it('should return NotImplemented error when object restore Tier is \'Bulk\'', done => { + it("should return NotImplemented error when object restore Tier is 'Bulk'", done => { mdColdHelper.putBucketMock(bucketName, null, () => { mdColdHelper.putObjectMock(bucketName, objectName, mdColdHelper.getArchivedObjectMD(), () => { objectRestore(authInfo, objectRestoreRequest(objectRestoreXmlBulkTier), log, err => { @@ -98,7 +104,7 @@ describe('restoreObject API', () => { }); }); - it('should return NotImplemented error when object restore Tier is \'Expedited\'', done => { + it("should return NotImplemented error when object restore Tier is 'Expedited'", done => { mdColdHelper.putBucketMock(bucketName, null, () => { mdColdHelper.putObjectMock(bucketName, objectName, mdColdHelper.getArchivedObjectMD(), () => { objectRestore(authInfo, objectRestoreRequest(objectRestoreXmlExpeditedTier), log, err => { @@ -109,48 +115,53 @@ describe('restoreObject API', () => { }); }); - it('should return Accepted and update objectMD ' + - 'while restoring an object from cold storage ' + - 'and the object doesn\'t have a restored copy in bucket', done => { - const testStartTime = new Date(Date.now()); - mdColdHelper.putBucketMock(bucketName, null, () => { - mdColdHelper.putObjectMock(bucketName, objectName, mdColdHelper.getArchivedObjectMD(), () => { - objectRestore(authInfo, objectRestoreRequest(objectRestoreXml), log, (err, statusCode) => { - assert.ifError(err); - assert.strictEqual(statusCode, 202); - metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => { - const testEndTime = new Date(Date.now()); - assert.strictEqual(md.archive.restoreRequestedDays, restoreDays); - assert.strictEqual(testStartTime < md.archive.restoreRequestedAt < testEndTime, true); - done(); + it( + 'should return Accepted and update objectMD ' + + 'while restoring an object from cold storage ' + + "and the object doesn't have a restored copy in bucket", + done => { + const testStartTime = new Date(Date.now()); + mdColdHelper.putBucketMock(bucketName, null, () => { + mdColdHelper.putObjectMock(bucketName, objectName, mdColdHelper.getArchivedObjectMD(), () => { + objectRestore(authInfo, objectRestoreRequest(objectRestoreXml), log, (err, statusCode) => { + assert.ifError(err); + assert.strictEqual(statusCode, 202); + metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => { + const testEndTime = new Date(Date.now()); + assert.strictEqual(md.archive.restoreRequestedDays, restoreDays); + assert.strictEqual(testStartTime < md.archive.restoreRequestedAt < testEndTime, true); + done(); }); + }); }); }); - }); - }); - - it('should update the expiry time and return OK ' + - 'while restoring an object from cold storage ' + - 'and the object have a restored copy in bucket', done => { - const testStartTime = new Date(Date.now()); - mdColdHelper.putBucketMock(bucketName, null, () => { - mdColdHelper.putObjectMock(bucketName, objectName, mdColdHelper.getRestoredObjectMD(), () => { - objectRestore(authInfo, objectRestoreRequest(objectRestoreXml), log, (err, statusCode) => { - assert.ifError(err); - assert.strictEqual(statusCode, 200); - metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => { - const testEndTime = new Date(Date.now()); - assert.strictEqual(md.archive.restoreRequestedDays, restoreDays); - assert.strictEqual(testStartTime < md.archive.restoreRequestedAt < testEndTime, true); + }, + ); + + it( + 'should update the expiry time and return OK ' + + 'while restoring an object from cold storage ' + + 'and the object have a restored copy in bucket', + done => { + const testStartTime = new Date(Date.now()); + mdColdHelper.putBucketMock(bucketName, null, () => { + mdColdHelper.putObjectMock(bucketName, objectName, mdColdHelper.getRestoredObjectMD(), () => { + objectRestore(authInfo, objectRestoreRequest(objectRestoreXml), log, (err, statusCode) => { + assert.ifError(err); + assert.strictEqual(statusCode, 200); + metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => { + const testEndTime = new Date(Date.now()); + assert.strictEqual(md.archive.restoreRequestedDays, restoreDays); + assert.strictEqual(testStartTime < md.archive.restoreRequestedAt < testEndTime, true); done(); }); + }); }); }); - }); - }); + }, + ); - it('should return InvalidObjectState ' + - 'while restoring an expired restored object', () => { + it('should return InvalidObjectState ' + 'while restoring an expired restored object', () => { mdColdHelper.putBucketMock(bucketName, null, () => { mdColdHelper.putObjectMock(bucketName, objectName, mdColdHelper.getExpiredObjectMD(), () => { objectRestore(authInfo, objectRestoreRequest(objectRestoreXml), log, err => { @@ -176,8 +187,11 @@ describe('restoreObject API', () => { try { assert(err, 'Expected an error'); assert.strictEqual(err.is.NoSuchKey, true); - assert.strictEqual(typeof err.customizeDescription, 'function', - 'Error should be from errorInstances which has customizeDescription method'); + assert.strictEqual( + typeof err.customizeDescription, + 'function', + 'Error should be from errorInstances which has customizeDescription method', + ); done(); } catch (assertionError) { done(assertionError); @@ -204,8 +218,11 @@ describe('restoreObject API', () => { try { assert(err, 'Expected an error'); assert.strictEqual(err.is.MethodNotAllowed, true); - assert.strictEqual(typeof err.customizeDescription, 'function', - 'Error should be from errorInstances which has customizeDescription method'); + assert.strictEqual( + typeof err.customizeDescription, + 'function', + 'Error should be from errorInstances which has customizeDescription method', + ); done(); } catch (assertionError) { done(assertionError); diff --git a/tests/unit/api/parseLikeExpression.js b/tests/unit/api/parseLikeExpression.js index 469b6a8df7..20b853c0de 100644 --- a/tests/unit/api/parseLikeExpression.js +++ b/tests/unit/api/parseLikeExpression.js @@ -1,6 +1,5 @@ const assert = require('assert'); -const parseLikeExpression = - require('../../../lib/api/apiUtils/bucket/parseLikeExpression'); +const parseLikeExpression = require('../../../lib/api/apiUtils/bucket/parseLikeExpression'); describe('parseLikeExpression', () => { const tests = [ @@ -29,11 +28,12 @@ describe('parseLikeExpression', () => { output: { $regex: /\//, $options: '' }, }, ]; - tests.forEach(test => it('should return correct MongoDB query object: ' + - `"${test.input}" => ${JSON.stringify(test.output)}`, () => { - const res = parseLikeExpression(test.input); - assert.deepStrictEqual(res, test.output); - })); + tests.forEach(test => + it('should return correct MongoDB query object: ' + `"${test.input}" => ${JSON.stringify(test.output)}`, () => { + const res = parseLikeExpression(test.input); + assert.deepStrictEqual(res, test.output); + }), + ); const badInputTests = [ { input: null, @@ -44,10 +44,10 @@ describe('parseLikeExpression', () => { output: null, }, ]; - badInputTests.forEach(test => it( - 'should return null if input is not a string ' + - `"${test.input}" => ${JSON.stringify(test.output)}`, () => { - const res = parseLikeExpression(test.input); - assert.deepStrictEqual(res, test.output); - })); + badInputTests.forEach(test => + it('should return null if input is not a string ' + `"${test.input}" => ${JSON.stringify(test.output)}`, () => { + const res = parseLikeExpression(test.input); + assert.deepStrictEqual(res, test.output); + }), + ); }); diff --git a/tests/unit/api/serviceGet.js b/tests/unit/api/serviceGet.js index 0804ac103d..054bd191c4 100644 --- a/tests/unit/api/serviceGet.js +++ b/tests/unit/api/serviceGet.js @@ -45,35 +45,33 @@ describe('serviceGet API', () => { url: '/', headers: { host: `${bucketName3}.s3.amazonaws.com` }, }; - async.waterfall([ - function waterfall1(next) { - bucketPut(authInfo, testbucketPutRequest1, log, next); + async.waterfall( + [ + function waterfall1(next) { + bucketPut(authInfo, testbucketPutRequest1, log, next); + }, + function waterfall2(corsHeaders, next) { + bucketPut(authInfo, testbucketPutRequest2, log, next); + }, + function waterfall3(corsHeaders, next) { + bucketPut(authInfo, testbucketPutRequest3, log, next); + }, + function waterfall4(corsHeaders, next) { + serviceGet(authInfo, serviceGetRequest, log, next); + }, + function waterfall4(result, next) { + parseString(result, next); + }, + ], + (err, result) => { + assert.strictEqual(result.ListAllMyBucketsResult.Buckets[0].Bucket.length, 3); + assert.strictEqual(result.ListAllMyBucketsResult.Buckets[0].Bucket[0].Name[0], bucketName1); + assert.strictEqual(result.ListAllMyBucketsResult.Buckets[0].Bucket[1].Name[0], bucketName2); + assert.strictEqual(result.ListAllMyBucketsResult.Buckets[0].Bucket[2].Name[0], bucketName3); + assert.strictEqual(result.ListAllMyBucketsResult.$.xmlns, 'http://s3.amazonaws.com/doc/2006-03-01/'); + done(); }, - function waterfall2(corsHeaders, next) { - bucketPut(authInfo, testbucketPutRequest2, log, next); - }, - function waterfall3(corsHeaders, next) { - bucketPut(authInfo, testbucketPutRequest3, log, next); - }, - function waterfall4(corsHeaders, next) { - serviceGet(authInfo, serviceGetRequest, log, next); - }, - function waterfall4(result, next) { - parseString(result, next); - }, - ], (err, result) => { - assert.strictEqual(result.ListAllMyBucketsResult - .Buckets[0].Bucket.length, 3); - assert.strictEqual(result.ListAllMyBucketsResult - .Buckets[0].Bucket[0].Name[0], bucketName1); - assert.strictEqual(result.ListAllMyBucketsResult - .Buckets[0].Bucket[1].Name[0], bucketName2); - assert.strictEqual(result.ListAllMyBucketsResult - .Buckets[0].Bucket[2].Name[0], bucketName3); - assert.strictEqual(result.ListAllMyBucketsResult.$.xmlns, - 'http://s3.amazonaws.com/doc/2006-03-01/'); - done(); - }); + ); }); it('should prevent anonymous user from accessing getService API', done => { diff --git a/tests/unit/api/transientBucket.js b/tests/unit/api/transientBucket.js index 9ddf6952db..1890e88576 100644 --- a/tests/unit/api/transientBucket.js +++ b/tests/unit/api/transientBucket.js @@ -14,15 +14,12 @@ const bucketPutWebsite = require('../../../lib/api/bucketPutWebsite'); const bucketDelete = require('../../../lib/api/bucketDelete'); const bucketDeleteCors = require('../../../lib/api/bucketDeleteCors'); const bucketDeleteWebsite = require('../../../lib/api/bucketDeleteWebsite'); -const completeMultipartUpload - = require('../../../lib/api/completeMultipartUpload'); +const completeMultipartUpload = require('../../../lib/api/completeMultipartUpload'); const { config } = require('../../../lib/Config'); const constants = require('../../../constants'); const DummyRequest = require('../DummyRequest'); -const initiateMultipartUpload - = require('../../../lib/api/initiateMultipartUpload'); -const { cleanup, createAlteredRequest, DummyRequestLogger, makeAuthInfo } - = require('../helpers'); +const initiateMultipartUpload = require('../../../lib/api/initiateMultipartUpload'); +const { cleanup, createAlteredRequest, DummyRequestLogger, makeAuthInfo } = require('../helpers'); const listMultipartUploads = require('../../../lib/api/listMultipartUploads'); const listParts = require('../../../lib/api/listParts'); const metadata = require('../metadataswitch'); @@ -63,18 +60,15 @@ const serviceGetRequest = { const userBucketOwner = 'admin'; const creationDate = new Date().toJSON(); -const usersBucket = new BucketInfo(usersBucketName, - userBucketOwner, userBucketOwner, creationDate); +const usersBucket = new BucketInfo(usersBucketName, userBucketOwner, userBucketOwner, creationDate); const locationConstraint = 'us-east-1'; describe('transient bucket handling', () => { beforeEach(done => { cleanup(); - const bucketMD = new BucketInfo(bucketName, canonicalID, - authInfo.getAccountDisplayName(), creationDate); + const bucketMD = new BucketInfo(bucketName, canonicalID, authInfo.getAccountDisplayName(), creationDate); bucketMD.addTransientFlag(); - bucketMD.setSpecificAcl(otherAccountAuthInfo.getCanonicalID(), - 'WRITE_ACP'); + bucketMD.setSpecificAcl(otherAccountAuthInfo.getCanonicalID(), 'WRITE_ACP'); bucketMD.setLocationConstraint(locationConstraint); metadata.createBucket(bucketName, bucketMD, log, () => { metadata.createBucket(usersBucketName, usersBucket, log, () => { @@ -83,86 +77,108 @@ describe('transient bucket handling', () => { }); }); - it('putBucket request should complete creation of transient bucket if ' + - 'request is from same account that originally put', done => { - bucketPut(authInfo, baseTestRequest, log, err => { - assert.ifError(err); - serviceGet(authInfo, serviceGetRequest, log, (err, data) => { - parseString(data, (err, result) => { - assert.strictEqual(result.ListAllMyBucketsResult - .Buckets[0].Bucket.length, 1); - assert.strictEqual(result.ListAllMyBucketsResult - .Buckets[0].Bucket[0].Name[0], bucketName); - done(); + it( + 'putBucket request should complete creation of transient bucket if ' + + 'request is from same account that originally put', + done => { + bucketPut(authInfo, baseTestRequest, log, err => { + assert.ifError(err); + serviceGet(authInfo, serviceGetRequest, log, (err, data) => { + parseString(data, (err, result) => { + assert.strictEqual(result.ListAllMyBucketsResult.Buckets[0].Bucket.length, 1); + assert.strictEqual(result.ListAllMyBucketsResult.Buckets[0].Bucket[0].Name[0], bucketName); + done(); + }); }); }); - }); - }); + }, + ); - it('putBucket request should return error if ' + - 'transient bucket created by different account', done => { + it('putBucket request should return error if ' + 'transient bucket created by different account', done => { bucketPut(otherAccountAuthInfo, baseTestRequest, log, err => { assert.strictEqual(err.is.BucketAlreadyExists, true); - serviceGet(otherAccountAuthInfo, serviceGetRequest, - log, (err, data) => { - parseString(data, (err, result) => { - assert.strictEqual(result.ListAllMyBucketsResult - .Buckets[0], ''); - done(); - }); + serviceGet(otherAccountAuthInfo, serviceGetRequest, log, (err, data) => { + parseString(data, (err, result) => { + assert.strictEqual(result.ListAllMyBucketsResult.Buckets[0], ''); + done(); }); + }); }); }); - it('ACLs from clean up putBucket request should overwrite ACLs from ' + - 'original failed request that resulted in transient state', done => { - const alteredRequest = createAlteredRequest({ - 'x-amz-acl': 'public-read' }, 'headers', - baseTestRequest, baseTestRequest.headers); - bucketPut(authInfo, alteredRequest, log, err => { - assert.ifError(err); - metadata.getBucket(bucketName, log, (err, data) => { - assert.strictEqual(data._transient, false); - assert.strictEqual(data._acl.Canned, 'public-read'); - assert.strictEqual(data._owner, authInfo.getCanonicalID()); - done(); + it( + 'ACLs from clean up putBucket request should overwrite ACLs from ' + + 'original failed request that resulted in transient state', + done => { + const alteredRequest = createAlteredRequest( + { + 'x-amz-acl': 'public-read', + }, + 'headers', + baseTestRequest, + baseTestRequest.headers, + ); + bucketPut(authInfo, alteredRequest, log, err => { + assert.ifError(err); + metadata.getBucket(bucketName, log, (err, data) => { + assert.strictEqual(data._transient, false); + assert.strictEqual(data._acl.Canned, 'public-read'); + assert.strictEqual(data._owner, authInfo.getCanonicalID()); + done(); + }); }); - }); - }); + }, + ); - it('putBucketACL request should complete creation of transient bucket if ' + - 'request is from same account that originally put', done => { - const putACLRequest = createAlteredRequest({ - 'x-amz-acl': 'public-read' }, 'headers', - baseTestRequest, baseTestRequest.headers); - putACLRequest.url = '/?acl'; - putACLRequest.query = { acl: '' }; - bucketPutACL(authInfo, putACLRequest, log, err => { - assert.ifError(err); - metadata.getBucket(bucketName, log, (err, data) => { - assert.strictEqual(data._transient, false); - assert.strictEqual(data._acl.Canned, 'public-read'); - assert.strictEqual(data._owner, authInfo.getCanonicalID()); - done(); + it( + 'putBucketACL request should complete creation of transient bucket if ' + + 'request is from same account that originally put', + done => { + const putACLRequest = createAlteredRequest( + { + 'x-amz-acl': 'public-read', + }, + 'headers', + baseTestRequest, + baseTestRequest.headers, + ); + putACLRequest.url = '/?acl'; + putACLRequest.query = { acl: '' }; + bucketPutACL(authInfo, putACLRequest, log, err => { + assert.ifError(err); + metadata.getBucket(bucketName, log, (err, data) => { + assert.strictEqual(data._transient, false); + assert.strictEqual(data._acl.Canned, 'public-read'); + assert.strictEqual(data._owner, authInfo.getCanonicalID()); + done(); + }); }); - }); - }); + }, + ); - it('putBucketACL request should complete creation of transient bucket if ' + - 'request is from another authorized account', done => { - const putACLRequest = createAlteredRequest({ - 'x-amz-acl': 'public-read' }, 'headers', - baseTestRequest, baseTestRequest.headers); - bucketPutACL(otherAccountAuthInfo, putACLRequest, log, err => { - assert.ifError(err); - metadata.getBucket(bucketName, log, (err, data) => { - assert.strictEqual(data._transient, false); - assert.strictEqual(data._acl.Canned, 'public-read'); - assert.strictEqual(data._owner, authInfo.getCanonicalID()); - done(); + it( + 'putBucketACL request should complete creation of transient bucket if ' + + 'request is from another authorized account', + done => { + const putACLRequest = createAlteredRequest( + { + 'x-amz-acl': 'public-read', + }, + 'headers', + baseTestRequest, + baseTestRequest.headers, + ); + bucketPutACL(otherAccountAuthInfo, putACLRequest, log, err => { + assert.ifError(err); + metadata.getBucket(bucketName, log, (err, data) => { + assert.strictEqual(data._transient, false); + assert.strictEqual(data._acl.Canned, 'public-read'); + assert.strictEqual(data._owner, authInfo.getCanonicalID()); + done(); + }); }); - }); - }); + }, + ); describe('objectPut on a transient bucket', () => { const objName = 'objectName'; @@ -172,10 +188,8 @@ describe('transient bucket handling', () => { }); }); - it('objectPut request should complete creation of transient bucket', - done => { - const setUpRequest = createAlteredRequest({}, 'headers', - baseTestRequest, baseTestRequest.headers); + it('objectPut request should complete creation of transient bucket', done => { + const setUpRequest = createAlteredRequest({}, 'headers', baseTestRequest, baseTestRequest.headers); setUpRequest.objectKey = objName; const postBody = Buffer.from('I am a body', 'utf8'); const md5Hash = crypto.createHash('md5'); @@ -186,12 +200,11 @@ describe('transient bucket handling', () => { metadata.getBucket(bucketName, log, (err, data) => { assert.strictEqual(data._transient, false); assert.strictEqual(data._owner, authInfo.getCanonicalID()); - metadata.getObjectMD(bucketName, objName, {}, log, - (err, obj) => { - assert.ifError(err); - assert.strictEqual(obj['content-md5'], etag); - done(); - }); + metadata.getObjectMD(bucketName, objName, {}, log, (err, obj) => { + assert.ifError(err); + assert.strictEqual(obj['content-md5'], etag); + done(); + }); }); }); }); @@ -200,19 +213,15 @@ describe('transient bucket handling', () => { describe('initiateMultipartUpload on a transient bucket', () => { const objName = 'objectName'; after(done => { - metadata.deleteObjectMD(`${constants.mpuBucketPrefix}` + - `${bucketName}`, objName, {}, log, () => { - metadata.deleteBucket(`${constants.mpuBucketPrefix}` + - `${bucketName}`, log, () => { - done(); - }); + metadata.deleteObjectMD(`${constants.mpuBucketPrefix}` + `${bucketName}`, objName, {}, log, () => { + metadata.deleteBucket(`${constants.mpuBucketPrefix}` + `${bucketName}`, log, () => { + done(); }); + }); }); - it('initiateMultipartUpload request should complete ' + - 'creation of transient bucket', done => { - const initiateRequest = createAlteredRequest({}, 'headers', - baseTestRequest, baseTestRequest.headers); + it('initiateMultipartUpload request should complete ' + 'creation of transient bucket', done => { + const initiateRequest = createAlteredRequest({}, 'headers', baseTestRequest, baseTestRequest.headers); initiateRequest.objectKey = objName; initiateRequest.url = `/${objName}?uploads`; initiateMultipartUpload(authInfo, initiateRequest, log, err => { @@ -220,21 +229,22 @@ describe('transient bucket handling', () => { metadata.getBucket(bucketName, log, (err, data) => { assert.strictEqual(data._transient, false); assert.strictEqual(data._owner, authInfo.getCanonicalID()); - metadata.listObject(`${constants.mpuBucketPrefix}` + - `${bucketName}`, + metadata.listObject( + `${constants.mpuBucketPrefix}` + `${bucketName}`, { prefix: `overview${constants.splitter}${objName}` }, - log, (err, results) => { + log, + (err, results) => { assert.ifError(err); assert.strictEqual(results.Contents.length, 1); done(); - }); + }, + ); }); }); }); }); - it('deleteBucket request should delete transient bucket if ' + - 'request is from owner', done => { + it('deleteBucket request should delete transient bucket if ' + 'request is from owner', done => { bucketDelete(authInfo, baseTestRequest, log, err => { assert.ifError(err); metadata.getBucket(bucketName, log, err => { @@ -244,166 +254,140 @@ describe('transient bucket handling', () => { }); }); - it('deleteBucket request should return error if ' + - 'request is not from owner', done => { - bucketDelete(otherAccountAuthInfo, baseTestRequest, - log, err => { - assert.strictEqual(err.is.AccessDenied, true); - done(); - }); + it('deleteBucket request should return error if ' + 'request is not from owner', done => { + bucketDelete(otherAccountAuthInfo, baseTestRequest, log, err => { + assert.strictEqual(err.is.AccessDenied, true); + done(); + }); }); - it('bucketGet request on transient bucket should return NoSuchBucket' + - 'error', done => { - const bucketGetRequest = createAlteredRequest({}, 'headers', - baseTestRequest, baseTestRequest.headers); + it('bucketGet request on transient bucket should return NoSuchBucket' + 'error', done => { + const bucketGetRequest = createAlteredRequest({}, 'headers', baseTestRequest, baseTestRequest.headers); bucketGetRequest.url = `/${bucketName}`; bucketGetRequest.query = {}; - bucketGet(authInfo, bucketGetRequest, - log, err => { - assert.strictEqual(err.is.NoSuchBucket, true); - done(); - }); + bucketGet(authInfo, bucketGetRequest, log, err => { + assert.strictEqual(err.is.NoSuchBucket, true); + done(); + }); }); - it('bucketGetACL request on transient bucket should return NoSuchBucket' + - 'error', done => { - const bucketGetACLRequest = createAlteredRequest({}, 'headers', - baseTestRequest, baseTestRequest.headers); + it('bucketGetACL request on transient bucket should return NoSuchBucket' + 'error', done => { + const bucketGetACLRequest = createAlteredRequest({}, 'headers', baseTestRequest, baseTestRequest.headers); bucketGetACLRequest.url = '/?acl'; bucketGetACLRequest.query = { acl: '' }; - bucketGetACL(authInfo, bucketGetACLRequest, - log, err => { - assert.strictEqual(err.is.NoSuchBucket, true); - done(); - }); + bucketGetACL(authInfo, bucketGetACLRequest, log, err => { + assert.strictEqual(err.is.NoSuchBucket, true); + done(); + }); }); - it('bucketGetCors request on transient bucket should return ' + - 'NoSuchBucket error', done => { + it('bucketGetCors request on transient bucket should return ' + 'NoSuchBucket error', done => { bucketGetCors(authInfo, baseTestRequest, log, err => { assert.strictEqual(err.is.NoSuchBucket, true); done(); }); }); - it('bucketPutCors request on transient bucket should return ' + - 'NoSuchBucket error', done => { - const bucketPutCorsRequest = createAlteredRequest({}, 'headers', - baseTestRequest, baseTestRequest.headers); - bucketPutCorsRequest.post = '' + - 'PUT' + - 'http://www.example.com' + - ''; - bucketPutCorsRequest.headers['content-md5'] = crypto.createHash('md5') - .update(bucketPutCorsRequest.post, 'utf8').digest('base64'); + it('bucketPutCors request on transient bucket should return ' + 'NoSuchBucket error', done => { + const bucketPutCorsRequest = createAlteredRequest({}, 'headers', baseTestRequest, baseTestRequest.headers); + bucketPutCorsRequest.post = + '' + + 'PUT' + + 'http://www.example.com' + + ''; + bucketPutCorsRequest.headers['content-md5'] = crypto + .createHash('md5') + .update(bucketPutCorsRequest.post, 'utf8') + .digest('base64'); bucketPutCors(authInfo, bucketPutCorsRequest, log, err => { assert.strictEqual(err.is.NoSuchBucket, true); done(); }); }); - it('bucketDeleteCors request on transient bucket should return ' + - 'NoSuchBucket error', done => { + it('bucketDeleteCors request on transient bucket should return ' + 'NoSuchBucket error', done => { bucketDeleteCors(authInfo, baseTestRequest, log, err => { assert.strictEqual(err.is.NoSuchBucket, true); done(); }); }); - it('bucketGetWebsite request on transient bucket should return ' + - 'NoSuchBucket error', done => { + it('bucketGetWebsite request on transient bucket should return ' + 'NoSuchBucket error', done => { bucketGetWebsite(authInfo, baseTestRequest, log, err => { assert.strictEqual(err.is.NoSuchBucket, true); done(); }); }); - it('bucketPutWebsite request on transient bucket should return ' + - 'NoSuchBucket error', done => { - const bucketPutWebsiteRequest = createAlteredRequest({}, 'headers', - baseTestRequest, baseTestRequest.headers); - bucketPutWebsiteRequest.post = '' + - 'index.html' + - ''; + it('bucketPutWebsite request on transient bucket should return ' + 'NoSuchBucket error', done => { + const bucketPutWebsiteRequest = createAlteredRequest({}, 'headers', baseTestRequest, baseTestRequest.headers); + bucketPutWebsiteRequest.post = + '' + + 'index.html' + + ''; bucketPutWebsite(authInfo, bucketPutWebsiteRequest, log, err => { assert.strictEqual(err.is.NoSuchBucket, true); done(); }); }); - it('bucketDeleteWebsite request on transient bucket should return ' + - 'NoSuchBucket error', done => { + it('bucketDeleteWebsite request on transient bucket should return ' + 'NoSuchBucket error', done => { bucketDeleteWebsite(authInfo, baseTestRequest, log, err => { assert.strictEqual(err.is.NoSuchBucket, true); done(); }); }); - it('bucketHead request on transient bucket should return NoSuchBucket' + - 'error', done => { - bucketHead(authInfo, baseTestRequest, - log, err => { - assert.strictEqual(err.is.NoSuchBucket, true); - done(); - }); + it('bucketHead request on transient bucket should return NoSuchBucket' + 'error', done => { + bucketHead(authInfo, baseTestRequest, log, err => { + assert.strictEqual(err.is.NoSuchBucket, true); + done(); + }); }); - it('completeMultipartUpload request on transient bucket should ' + - 'return NoSuchUpload error', done => { - const completeMpuRequest = createAlteredRequest({}, 'headers', - baseTestRequest, baseTestRequest.headers); + it('completeMultipartUpload request on transient bucket should ' + 'return NoSuchUpload error', done => { + const completeMpuRequest = createAlteredRequest({}, 'headers', baseTestRequest, baseTestRequest.headers); const uploadId = '5555'; completeMpuRequest.objectKey = 'objectName'; completeMpuRequest.query = { uploadId }; - completeMultipartUpload(authInfo, completeMpuRequest, - log, err => { - assert.strictEqual(err.is.NoSuchUpload, true); - done(); - }); + completeMultipartUpload(authInfo, completeMpuRequest, log, err => { + assert.strictEqual(err.is.NoSuchUpload, true); + done(); + }); }); - it('listParts request on transient bucket should ' + - 'return NoSuchUpload error', done => { - const listRequest = createAlteredRequest({}, 'headers', - baseTestRequest, baseTestRequest.headers); + it('listParts request on transient bucket should ' + 'return NoSuchUpload error', done => { + const listRequest = createAlteredRequest({}, 'headers', baseTestRequest, baseTestRequest.headers); const uploadId = '5555'; listRequest.objectKey = 'objectName'; listRequest.query = { uploadId }; - listParts(authInfo, listRequest, - log, err => { - assert.strictEqual(err.is.NoSuchUpload, true); - done(); - }); + listParts(authInfo, listRequest, log, err => { + assert.strictEqual(err.is.NoSuchUpload, true); + done(); + }); }); describe('multipartDelete request on a transient bucket', () => { - const deleteRequest = createAlteredRequest({}, 'headers', - baseTestRequest, baseTestRequest.headers); + const deleteRequest = createAlteredRequest({}, 'headers', baseTestRequest, baseTestRequest.headers); const uploadId = '5555'; deleteRequest.objectKey = 'objectName'; deleteRequest.query = { uploadId }; - const originalLegacyAWSBehavior = - config.locationConstraints[locationConstraint].legacyAwsBehavior; + const originalLegacyAWSBehavior = config.locationConstraints[locationConstraint].legacyAwsBehavior; after(done => { - config.locationConstraints[locationConstraint].legacyAwsBehavior = - originalLegacyAWSBehavior; + config.locationConstraints[locationConstraint].legacyAwsBehavior = originalLegacyAWSBehavior; done(); }); - it('should return NoSuchUpload error if legacyAwsBehavior is enabled', - done => { - config.locationConstraints[locationConstraint]. - legacyAwsBehavior = true; + it('should return NoSuchUpload error if legacyAwsBehavior is enabled', done => { + config.locationConstraints[locationConstraint].legacyAwsBehavior = true; multipartDelete(authInfo, deleteRequest, log, err => { assert.strictEqual(err.is.NoSuchUpload, true); done(); }); }); - it('should return no error if legacyAwsBehavior is not enabled', - done => { + it('should return no error if legacyAwsBehavior is not enabled', done => { config.locationConstraints[locationConstraint].legacyAwsBehavior = false; multipartDelete(authInfo, deleteRequest, log, err => { assert.ifError(err); @@ -412,75 +396,59 @@ describe('transient bucket handling', () => { }); }); - it('objectPutPart request on transient bucket should ' + - 'return NoSuchUpload error', done => { - const putPartRequest = createAlteredRequest({}, 'headers', - baseTestRequest, baseTestRequest.headers); + it('objectPutPart request on transient bucket should ' + 'return NoSuchUpload error', done => { + const putPartRequest = createAlteredRequest({}, 'headers', baseTestRequest, baseTestRequest.headers); const uploadId = '5555'; putPartRequest.objectKey = 'objectName'; putPartRequest.query = { uploadId, - partNumber: '1' }; - objectPutPart(authInfo, putPartRequest, undefined, - log, err => { - assert.strictEqual(err.is.NoSuchUpload, true); - done(); - }); + partNumber: '1', + }; + objectPutPart(authInfo, putPartRequest, undefined, log, err => { + assert.strictEqual(err.is.NoSuchUpload, true); + done(); + }); }); - it('list multipartUploads request on transient bucket should ' + - 'return NoSuchBucket error', done => { - const listRequest = createAlteredRequest({}, 'headers', - baseTestRequest, baseTestRequest.headers); + it('list multipartUploads request on transient bucket should ' + 'return NoSuchBucket error', done => { + const listRequest = createAlteredRequest({}, 'headers', baseTestRequest, baseTestRequest.headers); listRequest.query = {}; - listMultipartUploads(authInfo, listRequest, - log, err => { - assert.strictEqual(err.is.NoSuchBucket, true); - done(); - }); + listMultipartUploads(authInfo, listRequest, log, err => { + assert.strictEqual(err.is.NoSuchBucket, true); + done(); + }); }); - it('objectGet request on transient bucket should' + - 'return NoSuchBucket error', - done => { - objectGet(authInfo, baseTestRequest, false, - log, err => { - assert.strictEqual(err.is.NoSuchBucket, true); - done(); - }); + it('objectGet request on transient bucket should' + 'return NoSuchBucket error', done => { + objectGet(authInfo, baseTestRequest, false, log, err => { + assert.strictEqual(err.is.NoSuchBucket, true); + done(); }); + }); - it('objectGetACL request on transient bucket should return ' + - 'NoSuchBucket error', done => { - objectGetACL(authInfo, baseTestRequest, - log, err => { + it('objectGetACL request on transient bucket should return ' + 'NoSuchBucket error', done => { + objectGetACL(authInfo, baseTestRequest, log, err => { assert.strictEqual(err.is.NoSuchBucket, true); done(); }); }); - it('objectHead request on transient bucket should return ' + - 'NoSuchBucket error', done => { - objectHead(authInfo, baseTestRequest, - log, err => { + it('objectHead request on transient bucket should return ' + 'NoSuchBucket error', done => { + objectHead(authInfo, baseTestRequest, log, err => { assert.strictEqual(err.is.NoSuchBucket, true); done(); }); }); - it('objectPutACL request on transient bucket should return ' + - 'NoSuchBucket error', done => { - objectPutACL(authInfo, baseTestRequest, - log, err => { + it('objectPutACL request on transient bucket should return ' + 'NoSuchBucket error', done => { + objectPutACL(authInfo, baseTestRequest, log, err => { assert.strictEqual(err.is.NoSuchBucket, true); done(); }); }); - it('objectDelete request on transient bucket should return ' + - 'NoSuchBucket error', done => { - objectDelete(authInfo, baseTestRequest, - log, err => { + it('objectDelete request on transient bucket should return ' + 'NoSuchBucket error', done => { + objectDelete(authInfo, baseTestRequest, log, err => { assert.strictEqual(err.is.NoSuchBucket, true); done(); }); diff --git a/tests/unit/api/utils/metadataMockColdStorage.js b/tests/unit/api/utils/metadataMockColdStorage.js index b1f6496418..ca14070009 100644 --- a/tests/unit/api/utils/metadataMockColdStorage.js +++ b/tests/unit/api/utils/metadataMockColdStorage.js @@ -4,9 +4,7 @@ const { DummyRequestLogger } = require('../../helpers'); const log = new DummyRequestLogger(); const { BucketInfo, ObjectMD, ObjectMDAmzRestore, ObjectMDArchive } = require('arsenal').models; -const { - LOCATION_NAME_DMF, -} = require('../../../constants'); +const { LOCATION_NAME_DMF } = require('../../../constants'); const defaultLocation = LOCATION_NAME_DMF; const defaultOwnerId = '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be'; @@ -30,7 +28,7 @@ const baseMd = { FULL_CONTROL: [], WRITE_ACP: [], READ: [], - READ_ACP: [] + READ_ACP: [], }, key: 'objectName', location: [ @@ -39,8 +37,8 @@ const baseMd = { size: 11, start: 0, dataStoreName: 'mem', - dataStoreETag: '1:be747eb4b75517bf6b3cf7c5fbb62f3a' - } + dataStoreETag: '1:be747eb4b75517bf6b3cf7c5fbb62f3a', + }, ], isDeleteMarker: false, tags: {}, @@ -53,13 +51,13 @@ const baseMd = { role: '', storageType: '', dataStoreVersionId: '', - isNFS: null + isNFS: null, }, dataStoreName: 'mem', originOp: 's3:ObjectCreated:Put', 'last-modified': '2022-05-10T08:31:51.878Z', 'md-model-version': 5, - 'x-amz-meta-test': 'some metadata' + 'x-amz-meta-test': 'some metadata', }; /** @@ -81,7 +79,8 @@ function putBucketMock(bucketName, location, cb) { null, null, null, - location); + location, + ); return metadata.createBucket(bucketName, bucket, log, cb); } @@ -117,9 +116,11 @@ function getTransitionInProgressObjectMD() { */ function getArchivedObjectMD() { return getTransitionInProgressObjectMD() - .setArchive(new ObjectMDArchive( - { foo: 0, bar: 'stuff' }, // opaque, can be anything... - )) + .setArchive( + new ObjectMDArchive( + { foo: 0, bar: 'stuff' }, // opaque, can be anything... + ), + ) .setDataStoreName(defaultLocation) .setAmzStorageClass(defaultLocation) .setTransitionInProgress(false) @@ -133,14 +134,8 @@ function getArchivedObjectMD() { function getRestoringObjectMD() { const archivedObjectMD = getArchivedObjectMD(); return archivedObjectMD - .setAmzRestore(new ObjectMDAmzRestore( - true, - )) - .setArchive(new ObjectMDArchive( - archivedObjectMD.getArchive().getArchiveInfo(), - new Date(Date.now() - 60), - 5, - )) + .setAmzRestore(new ObjectMDAmzRestore(true)) + .setArchive(new ObjectMDArchive(archivedObjectMD.getArchive().getArchiveInfo(), new Date(Date.now() - 60), 5)) .setOriginOp('s3:ObjectRestore:Post'); } @@ -155,21 +150,20 @@ function getRestoredObjectMD(date) { const expiryDate = date || new Date(restoreDate.getTime() + 1000 * 60 * 60 * 24 * restoreDays); const restoringObjectMD = getRestoringObjectMD(); - const restoreInfo = new ObjectMDAmzRestore( - false, - expiryDate, - ); + const restoreInfo = new ObjectMDAmzRestore(false, expiryDate); restoreInfo['content-md5'] = restoredEtag; return restoringObjectMD .setAmzRestore(restoreInfo) - .setArchive(new ObjectMDArchive( - restoringObjectMD.getArchive().getArchiveInfo(), - new Date(Date.now() - 60000), - restoreDays, - restoreDate, - expiryDate, - )) + .setArchive( + new ObjectMDArchive( + restoringObjectMD.getArchive().getArchiveInfo(), + new Date(Date.now() - 60000), + restoreDays, + restoreDate, + expiryDate, + ), + ) .setDataStoreName('mem') .setOriginOp('s3:ObjectRestore:Completed'); } diff --git a/tests/unit/auth/TrailingChecksumTransform.js b/tests/unit/auth/TrailingChecksumTransform.js index 9cae54e659..b1ae3fbbd8 100644 --- a/tests/unit/auth/TrailingChecksumTransform.js +++ b/tests/unit/auth/TrailingChecksumTransform.js @@ -9,11 +9,12 @@ const { DummyRequestLogger } = require('../helpers'); const log = new DummyRequestLogger(); // note this is not the correct checksum in objDataWithTrailingChecksum -const objDataWithTrailingChecksum = '10\r\n01234\r6789abcd\r\n\r\n' + - '2\r\n01\r\n' + - '1\r\n2\r\n' + - 'd\r\n3456789abcdef\r\n' + - '0\r\nchecksum:xyz=\r\n'; +const objDataWithTrailingChecksum = + '10\r\n01234\r6789abcd\r\n\r\n' + + '2\r\n01\r\n' + + '1\r\n2\r\n' + + 'd\r\n3456789abcdef\r\n' + + '0\r\nchecksum:xyz=\r\n'; const objDataWithoutTrailingChecksum = '01234\r6789abcd\r\n0123456789abcdef'; class ChunkedReader extends Readable { @@ -43,9 +44,7 @@ describe('TrailingChecksumTransform class', () => { trailingChecksumTransform.on('error', err => { assert.strictEqual(err, null); }); - const chunks = [ - Buffer.from(objDataWithTrailingChecksum), - ]; + const chunks = [Buffer.from(objDataWithTrailingChecksum)]; const chunkedReader = new ChunkedReader(chunks); chunkedReader.pipe(trailingChecksumTransform); const outputChunks = []; diff --git a/tests/unit/auth/V4Transform.js b/tests/unit/auth/V4Transform.js index daad1138df..419ba1e10d 100644 --- a/tests/unit/auth/V4Transform.js +++ b/tests/unit/auth/V4Transform.js @@ -7,8 +7,7 @@ const { DummyRequestLogger } = require('../helpers'); const log = new DummyRequestLogger(); const streamingV4Params = { accessKey: 'accessKey1', - signatureFromRequest: '2b8637632a997e06ee7b6c85d7' + - '147d2025e8f04d4374f4d7d7320de1618c7509', + signatureFromRequest: '2b8637632a997e06ee7b6c85d7' + '147d2025e8f04d4374f4d7d7320de1618c7509', region: 'us-east-1', scopeDate: '20170516', timestamp: '20170516T204738Z', @@ -33,15 +32,10 @@ describe('V4Transform class', () => { const v4Transform = new V4Transform(streamingV4Params, log, err => { assert.strictEqual(err, null); }); - const filler1 = '8;chunk-signature=51d2511f7c6887907dff20474d8db6' + - '7d557e5f515a6fa6a8466bb12f8833bcca\r\ncontents\r\n'; - const filler2 = '0;chunk-signature=c0eac24b7ce72141ec077df9753db' + - '4cc8b7991491806689da0395c8bd0231e48\r\n'; - const chunks = [ - Buffer.from(filler1), - Buffer.from(filler2), - null, - ]; + const filler1 = + '8;chunk-signature=51d2511f7c6887907dff20474d8db6' + '7d557e5f515a6fa6a8466bb12f8833bcca\r\ncontents\r\n'; + const filler2 = '0;chunk-signature=c0eac24b7ce72141ec077df9753db' + '4cc8b7991491806689da0395c8bd0231e48\r\n'; + const chunks = [Buffer.from(filler1), Buffer.from(filler2), null]; const authMe = new AuthMe(chunks); authMe.pipe(v4Transform); v4Transform.on('finish', () => { @@ -54,15 +48,10 @@ describe('V4Transform class', () => { assert(err); done(); }); - const filler1 = '8;chunk-signature=51d2511f7c6887907dff20474d8db6' + - '7d557e5f515a6fa6a8466bb12f8833bcca\r\ncontents\r\n'; - const filler2 = '0;chunk-signature=baadc0debaadc0debaadc0debaadc0de' + - 'baadc0debaadc0debaadc0debaadc0de\r\n'; - const chunks = [ - Buffer.from(filler1), - Buffer.from(filler2), - null, - ]; + const filler1 = + '8;chunk-signature=51d2511f7c6887907dff20474d8db6' + '7d557e5f515a6fa6a8466bb12f8833bcca\r\ncontents\r\n'; + const filler2 = '0;chunk-signature=baadc0debaadc0debaadc0debaadc0de' + 'baadc0debaadc0debaadc0debaadc0de\r\n'; + const chunks = [Buffer.from(filler1), Buffer.from(filler2), null]; const authMe = new AuthMe(chunks); authMe.pipe(v4Transform); }); @@ -71,17 +60,11 @@ describe('V4Transform class', () => { const v4Transform = new V4Transform(streamingV4Params, log, () => { assert(false); }); - const filler1 = '8;chunk-signature=51d2511f7c6887907dff20474d8db6' + - '7d557e5f515a6fa6a8466bb12f8833bcca\r\ncontents\r\n'; - const filler2 = '0;chunk-signature=c0eac24b7ce72141ec077df9753db' + - '4cc8b7991491806689da0395c8bd0231e48\r\n'; + const filler1 = + '8;chunk-signature=51d2511f7c6887907dff20474d8db6' + '7d557e5f515a6fa6a8466bb12f8833bcca\r\ncontents\r\n'; + const filler2 = '0;chunk-signature=c0eac24b7ce72141ec077df9753db' + '4cc8b7991491806689da0395c8bd0231e48\r\n'; const filler3 = '\r\n'; - const chunks = [ - Buffer.from(filler1), - Buffer.from(filler2), - Buffer.from(filler3), - null, - ]; + const chunks = [Buffer.from(filler1), Buffer.from(filler2), Buffer.from(filler3), null]; const authMe = new AuthMe(chunks); authMe.pipe(v4Transform); v4Transform.on('finish', () => { diff --git a/tests/unit/auth/in_memory/backend.js b/tests/unit/auth/in_memory/backend.js index bc31066158..102f9ddabe 100644 --- a/tests/unit/auth/in_memory/backend.js +++ b/tests/unit/auth/in_memory/backend.js @@ -1,7 +1,6 @@ const assert = require('assert'); -const { buildAuthDataAccount } = - require('../../../../lib/auth/in_memory/builder'); +const { buildAuthDataAccount } = require('../../../../lib/auth/in_memory/builder'); const fakeAccessKey = 'fakeaccesskey'; const fakeSecretKey = 'fakesecretkey'; @@ -16,18 +15,20 @@ function getFirstAndOnlyAccount(authdata) { } describe('buildAuthDataAccount function', () => { - it('should return authdata with the default user name if no user ' + - 'name provided', () => { - const authdata = buildAuthDataAccount(fakeAccessKey, fakeSecretKey, - fakeCanonicalId, fakeServiceName); + it('should return authdata with the default user name if no user ' + 'name provided', () => { + const authdata = buildAuthDataAccount(fakeAccessKey, fakeSecretKey, fakeCanonicalId, fakeServiceName); const firstAccount = getFirstAndOnlyAccount(authdata); assert.strictEqual(firstAccount.name, defaultUserName); }); - it('should return authdata with the user name that has been ' + - 'provided', () => { - const authdata = buildAuthDataAccount(fakeAccessKey, fakeSecretKey, - fakeCanonicalId, fakeServiceName, fakeUserName); + it('should return authdata with the user name that has been ' + 'provided', () => { + const authdata = buildAuthDataAccount( + fakeAccessKey, + fakeSecretKey, + fakeCanonicalId, + fakeServiceName, + fakeUserName, + ); const firstAccount = getFirstAndOnlyAccount(authdata); assert.strictEqual(firstAccount.name, fakeUserName); }); diff --git a/tests/unit/auth/permissionChecks.js b/tests/unit/auth/permissionChecks.js index 69aaa5b9f9..703422958d 100644 --- a/tests/unit/auth/permissionChecks.js +++ b/tests/unit/auth/permissionChecks.js @@ -25,35 +25,50 @@ describe('checkBucketAcls', () => { { description: 'should return true if bucket owner matches canonicalID', input: { - bucketAcl: {}, requestType: 'anyType', canonicalID: 'ownerId', mainApiCall: 'anyApiCall', + bucketAcl: {}, + requestType: 'anyType', + canonicalID: 'ownerId', + mainApiCall: 'anyApiCall', }, expected: true, }, { description: 'should return true for objectGetTagging when mainApiCall is objectGet', input: { - bucketAcl: {}, requestType: 'objectGetTagging', canonicalID: 'anyId', mainApiCall: 'objectGet', + bucketAcl: {}, + requestType: 'objectGetTagging', + canonicalID: 'anyId', + mainApiCall: 'objectGet', }, expected: true, }, { description: 'should return true for objectPutTagging when mainApiCall is objectPut', input: { - bucketAcl: {}, requestType: 'objectPutTagging', canonicalID: 'anyId', mainApiCall: 'objectPut', + bucketAcl: {}, + requestType: 'objectPutTagging', + canonicalID: 'anyId', + mainApiCall: 'objectPut', }, expected: true, }, { description: 'should return true for objectPutLegalHold when mainApiCall is objectPut', input: { - bucketAcl: {}, requestType: 'objectPutLegalHold', canonicalID: 'anyId', mainApiCall: 'objectPut', + bucketAcl: {}, + requestType: 'objectPutLegalHold', + canonicalID: 'anyId', + mainApiCall: 'objectPut', }, expected: true, }, { description: 'should return true for objectPutRetention when mainApiCall is objectPut', input: { - bucketAcl: {}, requestType: 'objectPutRetention', canonicalID: 'anyId', mainApiCall: 'objectPut', + bucketAcl: {}, + requestType: 'objectPutRetention', + canonicalID: 'anyId', + mainApiCall: 'objectPut', }, expected: true, }, @@ -62,7 +77,10 @@ describe('checkBucketAcls', () => { input: { bucketAcl: { Canned: 'public-read-write', - }, requestType: 'initiateMultipartUpload', canonicalID: 'any', mainApiCall: 'initiateMultipartUpload', + }, + requestType: 'initiateMultipartUpload', + canonicalID: 'any', + mainApiCall: 'initiateMultipartUpload', }, expected: true, }, @@ -71,7 +89,10 @@ describe('checkBucketAcls', () => { input: { bucketAcl: { Canned: 'public-read-write', - }, requestType: 'objectPutPart', canonicalID: 'any', mainApiCall: 'objectPutPart', + }, + requestType: 'objectPutPart', + canonicalID: 'any', + mainApiCall: 'objectPutPart', }, expected: true, }, @@ -80,7 +101,10 @@ describe('checkBucketAcls', () => { input: { bucketAcl: { Canned: 'public-read-write', - }, requestType: 'completeMultipartUpload', canonicalID: 'any', mainApiCall: 'completeMultipartUpload', + }, + requestType: 'completeMultipartUpload', + canonicalID: 'any', + mainApiCall: 'completeMultipartUpload', }, expected: true, }, @@ -240,8 +264,12 @@ describe('checkBucketAcls', () => { // Mock the bucket based on the test scenario's input mockBucket.getAcl = () => scenario.input.bucketAcl; - const result = checkBucketAcls(mockBucket, - scenario.input.requestType, scenario.input.canonicalID, scenario.input.mainApiCall); + const result = checkBucketAcls( + mockBucket, + scenario.input.requestType, + scenario.input.canonicalID, + scenario.input.mainApiCall, + ); assert.strictEqual(result, scenario.expected); }); }); @@ -255,7 +283,7 @@ describe('checkObjectAcls', () => { }; const mockObjectMD = { 'owner-id': 'objectOwnerId', - 'acl': { + acl: { Canned: '', FULL_CONTROL: [], READ: [], @@ -266,42 +294,73 @@ describe('checkObjectAcls', () => { }; it('should return true if request type is in bucketOwnerActions and bucket owner matches canonicalID', () => { - assert.strictEqual(checkObjectAcls(mockBucket, mockObjectMD, bucketOwnerActions[0], - 'bucketOwnerId', false, false, 'anyApiCall'), true); + assert.strictEqual( + checkObjectAcls( + mockBucket, + mockObjectMD, + bucketOwnerActions[0], + 'bucketOwnerId', + false, + false, + 'anyApiCall', + ), + true, + ); }); it('should return true if objectMD owner matches canonicalID', () => { - assert.strictEqual(checkObjectAcls(mockBucket, mockObjectMD, 'anyType', - 'objectOwnerId', false, false, 'anyApiCall'), true); + assert.strictEqual( + checkObjectAcls(mockBucket, mockObjectMD, 'anyType', 'objectOwnerId', false, false, 'anyApiCall'), + true, + ); }); it('should return true for objectGetTagging when mainApiCall is objectGet and conditions met', () => { - assert.strictEqual(checkObjectAcls(mockBucket, mockObjectMD, 'objectGetTagging', - 'anyIdNotPublic', true, true, 'objectGet'), true); + assert.strictEqual( + checkObjectAcls(mockBucket, mockObjectMD, 'objectGetTagging', 'anyIdNotPublic', true, true, 'objectGet'), + true, + ); }); it('should return false if no acl provided in objectMD', () => { const objMDWithoutAcl = Object.assign({}, mockObjectMD); delete objMDWithoutAcl.acl; - assert.strictEqual(checkObjectAcls(mockBucket, objMDWithoutAcl, 'anyType', - 'anyId', false, false, 'anyApiCall'), false); + assert.strictEqual( + checkObjectAcls(mockBucket, objMDWithoutAcl, 'anyType', 'anyId', false, false, 'anyApiCall'), + false, + ); }); const tests = [ { - acl: 'public-read', reqType: 'objectGet', id: 'anyIdNotPublic', expected: true, + acl: 'public-read', + reqType: 'objectGet', + id: 'anyIdNotPublic', + expected: true, }, { - acl: 'public-read-write', reqType: 'objectGet', id: 'anyIdNotPublic', expected: true, + acl: 'public-read-write', + reqType: 'objectGet', + id: 'anyIdNotPublic', + expected: true, }, { - acl: 'authenticated-read', reqType: 'objectGet', id: 'anyIdNotPublic', expected: true, + acl: 'authenticated-read', + reqType: 'objectGet', + id: 'anyIdNotPublic', + expected: true, }, { - acl: 'bucket-owner-read', reqType: 'objectGet', id: 'bucketOwnerId', expected: true, + acl: 'bucket-owner-read', + reqType: 'objectGet', + id: 'bucketOwnerId', + expected: true, }, { - acl: 'bucket-owner-full-control', reqType: 'objectGet', id: 'bucketOwnerId', expected: true, + acl: 'bucket-owner-full-control', + reqType: 'objectGet', + id: 'bucketOwnerId', + expected: true, }, { aclList: ['someId', 'anyIdNotPublic'], @@ -323,19 +382,31 @@ describe('checkObjectAcls', () => { { reqType: 'completeMultipartUpload', id: 'anyId', expected: true }, { reqType: 'objectDelete', id: 'anyId', expected: true }, { - aclList: ['anyId'], aclField: 'FULL_CONTROL', reqType: 'objectPutACL', id: 'anyId', expected: true, + aclList: ['anyId'], + aclField: 'FULL_CONTROL', + reqType: 'objectPutACL', + id: 'anyId', + expected: true, }, { - aclList: ['anyId'], aclField: 'FULL_CONTROL', reqType: 'objectGetACL', id: 'anyId', expected: true, + aclList: ['anyId'], + aclField: 'FULL_CONTROL', + reqType: 'objectGetACL', + id: 'anyId', + expected: true, }, { - acl: '', reqType: 'objectGet', id: 'randomId', expected: false, + acl: '', + reqType: 'objectGet', + id: 'randomId', + expected: false, }, ]; tests.forEach(test => { - it(`should return ${test.expected} for ${test.reqType} with ACL as ${test.acl - || (`${test.aclField}:${JSON.stringify(test.aclList)}`)}`, () => { + it(`should return ${test.expected} for ${test.reqType} with ACL as ${ + test.acl || `${test.aclField}:${JSON.stringify(test.aclList)}` + }`, () => { if (test.acl) { mockObjectMD.acl.Canned = test.acl; } else if (test.aclList && test.aclField) { @@ -360,112 +431,123 @@ describe('validatePolicyConditions', () => { { description: 'Should return null if conditions have a valid IP address', inputPolicy: { - Statement: [{ - Condition: { - IpAddress: { 'aws:SourceIp': '192.168.1.1/24' }, + Statement: [ + { + Condition: { + IpAddress: { 'aws:SourceIp': '192.168.1.1/24' }, + }, }, - }], + ], }, expected: null, }, { - description: 'Should return "Invalid IP address in Conditions" ' + - 'if conditions have an invalid IP address', + description: + 'Should return "Invalid IP address in Conditions" ' + 'if conditions have an invalid IP address', inputPolicy: { - Statement: [{ - Condition: { - IpAddress: { 'aws:SourceIp': '123' }, + Statement: [ + { + Condition: { + IpAddress: { 'aws:SourceIp': '123' }, + }, }, - }], + ], }, expected: 'Invalid IP address in Conditions', }, { - description: 'Should return "Policy has an invalid condition key" if a' + - ' condition key does not start with \'aws:\' and is not recognized', + description: + 'Should return "Policy has an invalid condition key" if a' + + " condition key does not start with 'aws:' and is not recognized", inputPolicy: { - Statement: [{ - Condition: { - NotARealCondition: { 's3:prefix': 'something' }, + Statement: [ + { + Condition: { + NotARealCondition: { 's3:prefix': 'something' }, + }, }, - }], + ], }, expected: 'Policy has an invalid condition key', }, { - description: 'Should return null if a statement in the policy does not contain a \'Condition\' block', + description: "Should return null if a statement in the policy does not contain a 'Condition' block", inputPolicy: { Statement: [{}], }, expected: null, }, { - description: 'Should return a relevant error message ' + - 'if the condition value is an empty string', + description: 'Should return a relevant error message ' + 'if the condition value is an empty string', inputPolicy: { - Statement: [{ - Condition: { - IpAddress: { 'aws:SourceIp': '' }, + Statement: [ + { + Condition: { + IpAddress: { 'aws:SourceIp': '' }, + }, }, - }], + ], }, expected: 'Invalid IP address in Conditions', }, { description: 'Should accept arrays of IPs', inputPolicy: { - Statement: [{ - Condition: { - IpAddress: { - 'aws:SourceIp': [ - '10.0.11.0/24', - '10.0.1.0/24', - ], + Statement: [ + { + Condition: { + IpAddress: { + 'aws:SourceIp': ['10.0.11.0/24', '10.0.1.0/24'], + }, }, }, - }], + ], }, expected: null, }, { description: 'Should return relevant error if one of the IPs in the array is invalid', inputPolicy: { - Statement: [{ - Condition: { - IpAddress: { - 'aws:SourceIp': [ - '10.0.11.0/24', - '123', - ], + Statement: [ + { + Condition: { + IpAddress: { + 'aws:SourceIp': ['10.0.11.0/24', '123'], + }, }, }, - }], + ], }, expected: 'Invalid IP address in Conditions', }, { description: 'Should not return error if array value in IP condition is empty', // this is AWS behavior inputPolicy: { - Statement: [{ - Condition: { - IpAddress: { - 'aws:SourceIp': [], + Statement: [ + { + Condition: { + IpAddress: { + 'aws:SourceIp': [], + }, }, }, - }], + ], }, expected: null, }, { - description: 'Should return null or a relevant error message ' + - 'if multiple conditions are provided in a single statement', + description: + 'Should return null or a relevant error message ' + + 'if multiple conditions are provided in a single statement', inputPolicy: { - Statement: [{ - Condition: { - IpAddress: { 'aws:SourceIp': '192.168.1.1' }, - NotARealCondition: { 's3:prefix': 'something' }, + Statement: [ + { + Condition: { + IpAddress: { 'aws:SourceIp': '192.168.1.1' }, + NotARealCondition: { 's3:prefix': 'something' }, + }, }, - }], + ], }, expected: 'Policy has an invalid condition key', }, @@ -490,34 +572,41 @@ describe('validatePolicyConditions', () => { { description: 'Should return null if conditions have a valid IPv6 address', inputPolicy: { - Statement: [{ - Condition: { - IpAddress: { 'aws:SourceIp': '2001:0db8:85a3:0000:0000:8a2e:0370:7334' }, + Statement: [ + { + Condition: { + IpAddress: { 'aws:SourceIp': '2001:0db8:85a3:0000:0000:8a2e:0370:7334' }, + }, }, - }], + ], }, expected: null, }, { description: 'Should return "Invalid IP address in Conditions" if conditions have an invalid IPv6 address', inputPolicy: { - Statement: [{ - Condition: { - IpAddress: { 'aws:SourceIp': '2001:0db8:85a3:0000:XYZZ:8a2e:0370:7334' }, + Statement: [ + { + Condition: { + IpAddress: { 'aws:SourceIp': '2001:0db8:85a3:0000:XYZZ:8a2e:0370:7334' }, + }, }, - }], + ], }, expected: 'Invalid IP address in Conditions', }, { - description: 'Should return "Invalid IP address in Conditions" if conditions' - + ' have an IPv6 address with unusual and invalid notation', + description: + 'Should return "Invalid IP address in Conditions" if conditions' + + ' have an IPv6 address with unusual and invalid notation', inputPolicy: { - Statement: [{ - Condition: { - IpAddress: { 'aws:SourceIp': '2001::85a3::8a2e' }, + Statement: [ + { + Condition: { + IpAddress: { 'aws:SourceIp': '2001::85a3::8a2e' }, + }, }, - }], + ], }, expected: 'Invalid IP address in Conditions', }, diff --git a/tests/unit/bucket/bucket_mem_api.js b/tests/unit/bucket/bucket_mem_api.js index 980a5ba83f..8c0a2f3efd 100644 --- a/tests/unit/bucket/bucket_mem_api.js +++ b/tests/unit/bucket/bucket_mem_api.js @@ -12,30 +12,25 @@ const bucketName = 'Zaphod'; const objMD = { test: '8' }; const log = new DummyRequestLogger(); -describe('bucket API for getting, putting and deleting ' + - 'objects in a bucket', () => { +describe('bucket API for getting, putting and deleting ' + 'objects in a bucket', () => { let bucket; before(done => { cleanup(); const creationDate = new Date().toJSON(); - bucket = new BucketInfo(bucketName, 'iAmTheOwnerId', - 'iAmTheOwnerDisplayName', creationDate); + bucket = new BucketInfo(bucketName, 'iAmTheOwnerId', 'iAmTheOwnerDisplayName', creationDate); metadata.createBucket(bucketName, bucket, log, done); }); - it('should be able to add an object to a bucket ' + - 'and get the object by key', done => { + it('should be able to add an object to a bucket ' + 'and get the object by key', done => { metadata.putObjectMD(bucketName, 'sampleKey', objMD, {}, log, () => { - metadata.getObjectMD(bucketName, 'sampleKey', {}, log, - (err, value) => { + metadata.getObjectMD(bucketName, 'sampleKey', {}, log, (err, value) => { assert.deepStrictEqual(value, objMD); done(); }); }); }); - it('should return an error in response ' + - 'to getObjectMD when no such key', done => { + it('should return an error in response ' + 'to getObjectMD when no such key', done => { metadata.getObjectMD(bucketName, 'notThere', {}, log, (err, value) => { assert.strictEqual(err.is.NoSuchKey, true); assert.strictEqual(value, undefined); @@ -44,22 +39,18 @@ describe('bucket API for getting, putting and deleting ' + }); it('should be able to delete an object from a bucket', done => { - metadata.putObjectMD(bucketName, 'objectToDelete', '{}', {}, log, - () => { - metadata.deleteObjectMD(bucketName, 'objectToDelete', {}, log, - () => { - metadata.getObjectMD(bucketName, 'objectToDelete', {}, log, - (err, value) => { - assert.strictEqual(err.is.NoSuchKey, true); - assert.strictEqual(value, undefined); - done(); - }); + metadata.putObjectMD(bucketName, 'objectToDelete', '{}', {}, log, () => { + metadata.deleteObjectMD(bucketName, 'objectToDelete', {}, log, () => { + metadata.getObjectMD(bucketName, 'objectToDelete', {}, log, (err, value) => { + assert.strictEqual(err.is.NoSuchKey, true); + assert.strictEqual(value, undefined); + done(); + }); }); }); }); }); - describe('bucket API for getting a subset of objects from a bucket', () => { /* * Implementation of AWS GET Bucket (List Objects) functionality @@ -104,147 +95,127 @@ describe('bucket API for getting a subset of objects from a bucket', () => { before(done => { cleanup(); const creationDate = new Date().toJSON(); - bucket = new BucketInfo(bucketName, 'ownerid', - 'ownerdisplayname', creationDate); + bucket = new BucketInfo(bucketName, 'ownerid', 'ownerdisplayname', creationDate); metadata.createBucket(bucketName, bucket, log, done); }); - it('should return individual key if key does not contain ' + - 'the delimiter even if key contains prefix', done => { - async.waterfall([ - next => - metadata.putObjectMD(bucketName, 'key1', '{}', {}, log, next), - (data, next) => - metadata.putObjectMD(bucketName, 'noMatchKey', '{}', {}, log, - next), - (data, next) => - metadata.putObjectMD(bucketName, 'key1/', '{}', {}, log, next), - (data, next) => - metadata.listObject(bucketName, { prefix: 'key', delimiter, - maxKeys: defaultLimit }, log, next), - ], (err, response) => { - assert.strictEqual(isKeyInContents(response, 'key1'), true); - assert.strictEqual(response.CommonPrefixes.indexOf('key1'), -1); - assert.strictEqual(isKeyInContents(response, 'key1/'), false); - assert(response.CommonPrefixes.indexOf('key1/') > -1); - assert.strictEqual(isKeyInContents(response, 'noMatchKey'), false); - assert.strictEqual(response.CommonPrefixes.indexOf('noMatchKey'), - -1); - done(); - }); - }); - - it('should return grouped keys under common prefix if keys start with ' + - 'given prefix and contain given delimiter', done => { - async.waterfall([ - next => - metadata.putObjectMD(bucketName, 'key/one', '{}', {}, log, - next), - (data, next) => - metadata.putObjectMD(bucketName, 'key/two', '{}', {}, log, - next), - (data, next) => - metadata.putObjectMD(bucketName, 'key/three', '{}', {}, log, - next), - (data, next) => - metadata.listObject(bucketName, { prefix: 'ke', delimiter, - maxKeys: defaultLimit }, log, next), - ], (err, response) => { - assert(response.CommonPrefixes.indexOf('key/') > -1); - assert.strictEqual(isKeyInContents(response, 'key/'), false); - done(); - }); + it('should return individual key if key does not contain ' + 'the delimiter even if key contains prefix', done => { + async.waterfall( + [ + next => metadata.putObjectMD(bucketName, 'key1', '{}', {}, log, next), + (data, next) => metadata.putObjectMD(bucketName, 'noMatchKey', '{}', {}, log, next), + (data, next) => metadata.putObjectMD(bucketName, 'key1/', '{}', {}, log, next), + (data, next) => + metadata.listObject(bucketName, { prefix: 'key', delimiter, maxKeys: defaultLimit }, log, next), + ], + (err, response) => { + assert.strictEqual(isKeyInContents(response, 'key1'), true); + assert.strictEqual(response.CommonPrefixes.indexOf('key1'), -1); + assert.strictEqual(isKeyInContents(response, 'key1/'), false); + assert(response.CommonPrefixes.indexOf('key1/') > -1); + assert.strictEqual(isKeyInContents(response, 'noMatchKey'), false); + assert.strictEqual(response.CommonPrefixes.indexOf('noMatchKey'), -1); + done(); + }, + ); }); - it('should return grouped keys if no prefix ' + - 'given and keys match before delimiter', done => { + it( + 'should return grouped keys under common prefix if keys start with ' + + 'given prefix and contain given delimiter', + done => { + async.waterfall( + [ + next => metadata.putObjectMD(bucketName, 'key/one', '{}', {}, log, next), + (data, next) => metadata.putObjectMD(bucketName, 'key/two', '{}', {}, log, next), + (data, next) => metadata.putObjectMD(bucketName, 'key/three', '{}', {}, log, next), + (data, next) => + metadata.listObject(bucketName, { prefix: 'ke', delimiter, maxKeys: defaultLimit }, log, next), + ], + (err, response) => { + assert(response.CommonPrefixes.indexOf('key/') > -1); + assert.strictEqual(isKeyInContents(response, 'key/'), false); + done(); + }, + ); + }, + ); + + it('should return grouped keys if no prefix ' + 'given and keys match before delimiter', done => { metadata.putObjectMD(bucketName, 'noPrefix/one', '{}', {}, log, () => { - metadata.putObjectMD(bucketName, 'noPrefix/two', '{}', {}, log, - () => { - metadata.listObject(bucketName, { delimiter, - maxKeys: defaultLimit }, log, (err, response) => { - assert(response.CommonPrefixes.indexOf('noPrefix/') - > -1); - assert.strictEqual(isKeyInContents(response, - 'noPrefix'), false); - done(); - }); + metadata.putObjectMD(bucketName, 'noPrefix/two', '{}', {}, log, () => { + metadata.listObject(bucketName, { delimiter, maxKeys: defaultLimit }, log, (err, response) => { + assert(response.CommonPrefixes.indexOf('noPrefix/') > -1); + assert.strictEqual(isKeyInContents(response, 'noPrefix'), false); + done(); + }); }); }); }); - it('should return no grouped keys if no ' + - 'delimiter specified in getBucketListObjects', done => { - metadata.listObject(bucketName, - { prefix: 'key', maxKeys: defaultLimit }, log, - (err, response) => { - assert.strictEqual(response.CommonPrefixes.length, 0); - done(); - }); + it('should return no grouped keys if no ' + 'delimiter specified in getBucketListObjects', done => { + metadata.listObject(bucketName, { prefix: 'key', maxKeys: defaultLimit }, log, (err, response) => { + assert.strictEqual(response.CommonPrefixes.length, 0); + done(); + }); }); - it('should only return keys occurring alphabetically ' + - 'AFTER marker when no delimiter specified', done => { + it('should only return keys occurring alphabetically ' + 'AFTER marker when no delimiter specified', done => { metadata.putObjectMD(bucketName, 'a', '{}', {}, log, () => { metadata.putObjectMD(bucketName, 'b', '{}', {}, log, () => { - metadata.listObject(bucketName, - { marker: 'a', maxKeys: defaultLimit }, - log, (err, response) => { - assert(isKeyInContents(response, 'b')); - assert.strictEqual(isKeyInContents(response, 'a'), - false); - done(); - }); + metadata.listObject(bucketName, { marker: 'a', maxKeys: defaultLimit }, log, (err, response) => { + assert(isKeyInContents(response, 'b')); + assert.strictEqual(isKeyInContents(response, 'a'), false); + done(); + }); }); }); }); - it('should only return keys occurring alphabetically AFTER ' + - 'marker when delimiter specified', done => { - metadata.listObject(bucketName, - { marker: 'a', delimiter, maxKeys: defaultLimit }, - log, (err, response) => { - assert(isKeyInContents(response, 'b')); - assert.strictEqual(isKeyInContents(response, 'a'), false); - done(); - }); + it('should only return keys occurring alphabetically AFTER ' + 'marker when delimiter specified', done => { + metadata.listObject(bucketName, { marker: 'a', delimiter, maxKeys: defaultLimit }, log, (err, response) => { + assert(isKeyInContents(response, 'b')); + assert.strictEqual(isKeyInContents(response, 'a'), false); + done(); + }); }); - it('should only return keys occurring alphabetically AFTER ' + - 'marker when delimiter and prefix specified', done => { - metadata.listObject(bucketName, - { prefix: 'b', marker: 'a', delimiter, maxKeys: defaultLimit }, - log, (err, response) => { - assert(isKeyInContents(response, 'b')); - assert.strictEqual(isKeyInContents(response, 'a'), false); - done(); - }); - }); + it( + 'should only return keys occurring alphabetically AFTER ' + 'marker when delimiter and prefix specified', + done => { + metadata.listObject( + bucketName, + { prefix: 'b', marker: 'a', delimiter, maxKeys: defaultLimit }, + log, + (err, response) => { + assert(isKeyInContents(response, 'b')); + assert.strictEqual(isKeyInContents(response, 'a'), false); + done(); + }, + ); + }, + ); // Next marker should be the last common prefix or contents key returned it('should return a NextMarker if maxKeys reached', done => { - async.waterfall([ - next => - metadata.putObjectMD(bucketName, 'next/', '{}', {}, log, next), - (data, next) => - metadata.putObjectMD(bucketName, 'next/rollUp', '{}', {}, log, - next), - (data, next) => - metadata.putObjectMD(bucketName, 'next1/', '{}', {}, log, next), - (data, next) => - metadata.listObject(bucketName, - { prefix: 'next', delimiter, maxKeys: smallLimit }, - log, next), - ], (err, response) => { - assert(response.CommonPrefixes.indexOf('next/') > -1); - assert.strictEqual(response.CommonPrefixes.indexOf('next1/'), -1); - assert.strictEqual(response.NextMarker, 'next/'); - assert(response.IsTruncated); - done(); - }); + async.waterfall( + [ + next => metadata.putObjectMD(bucketName, 'next/', '{}', {}, log, next), + (data, next) => metadata.putObjectMD(bucketName, 'next/rollUp', '{}', {}, log, next), + (data, next) => metadata.putObjectMD(bucketName, 'next1/', '{}', {}, log, next), + (data, next) => + metadata.listObject(bucketName, { prefix: 'next', delimiter, maxKeys: smallLimit }, log, next), + ], + (err, response) => { + assert(response.CommonPrefixes.indexOf('next/') > -1); + assert.strictEqual(response.CommonPrefixes.indexOf('next1/'), -1); + assert.strictEqual(response.NextMarker, 'next/'); + assert(response.IsTruncated); + done(); + }, + ); }); }); - describe('stress test for bucket API', function describe() { this.timeout(200000); @@ -270,86 +241,83 @@ describe('stress test for bucket API', function describe() { before(done => { cleanup(); const creationDate = new Date().toJSON(); - bucket = new BucketInfo(bucketName, 'ownerid', - 'ownerdisplayname', creationDate); + bucket = new BucketInfo(bucketName, 'ownerid', 'ownerdisplayname', creationDate); metadata.createBucket(bucketName, bucket, log, done); }); - it(`should put ${numKeys} keys into bucket and retrieve bucket list ` + - `in under ${maxMilliseconds} milliseconds`, done => { - const data = {}; - const keys = []; - - // Create dictionary entries based on prefixes array - for (let i = 0; i < prefixes.length; i++) { - data[prefixes[i]] = []; - } - // Populate dictionary with random key extensions - let prefix; - for (let j = 0; j < numKeys; j++) { - prefix = prefixes[j % prefixes.length]; - data[prefix].push(makeid(10)); - } - - // Populate keys array with all keys including prefixes - Object.keys(data).forEach(dkey => { - data[dkey].forEach(key => { - keys.push(dkey + delimiter + key); + it( + `should put ${numKeys} keys into bucket and retrieve bucket list ` + `in under ${maxMilliseconds} milliseconds`, + done => { + const data = {}; + const keys = []; + + // Create dictionary entries based on prefixes array + for (let i = 0; i < prefixes.length; i++) { + data[prefixes[i]] = []; + } + // Populate dictionary with random key extensions + let prefix; + for (let j = 0; j < numKeys; j++) { + prefix = prefixes[j % prefixes.length]; + data[prefix].push(makeid(10)); + } + + // Populate keys array with all keys including prefixes + Object.keys(data).forEach(dkey => { + data[dkey].forEach(key => { + keys.push(dkey + delimiter + key); + }); }); - }); - // Shuffle the keys array so the keys appear in random order - shuffle(keys); + // Shuffle the keys array so the keys appear in random order + shuffle(keys); - // Start timing - const startTime = process.hrtime(); + // Start timing + const startTime = process.hrtime(); - async.each(keys, (item, next) => { - metadata.putObjectMD(bucketName, item, '{}', {}, log, next); - }, err => { - if (err) { - assert.strictEqual(err, undefined); - done(); - } else { - metadata.listObject(bucketName, { delimiter }, - log, (err, response) => { - // Stop timing and calculate millisecond time difference - const diff = timeDiff(startTime); - assert(diff < maxMilliseconds); - prefixes.forEach(prefix => { - assert(response.CommonPrefixes - .indexOf(prefix + delimiter) > -1); - }); + async.each( + keys, + (item, next) => { + metadata.putObjectMD(bucketName, item, '{}', {}, log, next); + }, + err => { + if (err) { + assert.strictEqual(err, undefined); done(); - }); - } + } else { + metadata.listObject(bucketName, { delimiter }, log, (err, response) => { + // Stop timing and calculate millisecond time difference + const diff = timeDiff(startTime); + assert(diff < maxMilliseconds); + prefixes.forEach(prefix => { + assert(response.CommonPrefixes.indexOf(prefix + delimiter) > -1); + }); + done(); + }); + } + }, + ); + }, + ); + + it('should return all keys as Contents if delimiter ' + 'does not match and specify NextMarker', done => { + metadata.listObject(bucketName, { delimiter: oddDelimiter, maxKeys: testLimit }, log, (err, response) => { + assert.strictEqual(response.CommonPrefixes.length, 0); + assert.strictEqual(response.Contents.length, testLimit); + assert.strictEqual(response.IsTruncated, true); + assert.strictEqual(typeof response.NextMarker, 'string'); + done(); }); }); - it('should return all keys as Contents if delimiter ' + - 'does not match and specify NextMarker', done => { - metadata.listObject(bucketName, - { delimiter: oddDelimiter, maxKeys: testLimit }, - log, (err, response) => { - assert.strictEqual(response.CommonPrefixes.length, 0); - assert.strictEqual(response.Contents.length, testLimit); - assert.strictEqual(response.IsTruncated, true); - assert.strictEqual(typeof response.NextMarker, 'string'); - done(); - }); - }); - - it('should return only keys occurring ' + - 'after specified marker', done => { - metadata.listObject(bucketName, { marker: testMarker, delimiter }, log, - (err, res) => { - assert.strictEqual(res.CommonPrefixes.length, - prefixes.length - 1); - assert.strictEqual(res.CommonPrefixes.indexOf(testPrefix), -1); - assert.strictEqual(res.Contents.length, 0); - assert.strictEqual(res.IsTruncated, false); - assert.strictEqual(res.NextMarker, undefined); - done(); - }); + it('should return only keys occurring ' + 'after specified marker', done => { + metadata.listObject(bucketName, { marker: testMarker, delimiter }, log, (err, res) => { + assert.strictEqual(res.CommonPrefixes.length, prefixes.length - 1); + assert.strictEqual(res.CommonPrefixes.indexOf(testPrefix), -1); + assert.strictEqual(res.Contents.length, 0); + assert.strictEqual(res.IsTruncated, false); + assert.strictEqual(res.NextMarker, undefined); + done(); + }); }); }); diff --git a/tests/unit/encryption/checkHealth.js b/tests/unit/encryption/checkHealth.js index e6b5d90c64..4bad434042 100644 --- a/tests/unit/encryption/checkHealth.js +++ b/tests/unit/encryption/checkHealth.js @@ -61,10 +61,12 @@ describe('KMS.checkHealth', () => { assert(shouldRefreshStub.calledOnce, 'shouldRefresh should be called once'); - assert(setResultSpy.calledOnceWithExactly({ - code: 200, - message: 'OK', - })); + assert( + setResultSpy.calledOnceWithExactly({ + code: 200, + message: 'OK', + }), + ); done(); }); @@ -89,11 +91,13 @@ describe('KMS.checkHealth', () => { assert(shouldRefreshStub.calledOnce, 'shouldRefresh should be called once'); - assert(setResultSpy.calledOnceWithExactly({ - code: 500, - message: 'KMS health check failed', - description: 'We encountered an internal error. Please try again.', - })); + assert( + setResultSpy.calledOnceWithExactly({ + code: 500, + message: 'KMS health check failed', + description: 'We encountered an internal error. Please try again.', + }), + ); done(); }); diff --git a/tests/unit/encryption/healthCheckCache.js b/tests/unit/encryption/healthCheckCache.js index 4d21232e8f..8cb66cfa49 100644 --- a/tests/unit/encryption/healthCheckCache.js +++ b/tests/unit/encryption/healthCheckCache.js @@ -68,7 +68,7 @@ describe('Cache Class', () => { it('should return false if elapsed time is less than duration minus maximum jitter', () => { const fakeNow = 1625077800000; - const fakeLastChecked = fakeNow - (45 * 60 * 1000); // 45 minutes ago + const fakeLastChecked = fakeNow - 45 * 60 * 1000; // 45 minutes ago sandbox.stub(Date, 'now').returns(fakeNow); sandbox.stub(Math, 'random').returns(0); cache.lastChecked = fakeLastChecked; @@ -80,7 +80,7 @@ describe('Cache Class', () => { it('should return true if elapsed time is greater than duration minus maximum jitter', () => { const fakeNow = 1625077800000; - const fakeLastChecked = fakeNow - (61 * 60 * 1000); // 61 minutes ago + const fakeLastChecked = fakeNow - 61 * 60 * 1000; // 61 minutes ago sandbox.stub(Date, 'now').returns(fakeNow); sandbox.stub(Math, 'random').returns(0); cache.lastChecked = fakeLastChecked; @@ -96,7 +96,7 @@ describe('Cache Class', () => { sandbox.stub(Date, 'now').returns(fakeNow); // Elapsed time = 5 hours - const fakeLastChecked1 = fakeNow - (5 * 60 * 60 * 1000); + const fakeLastChecked1 = fakeNow - 5 * 60 * 60 * 1000; cache.lastChecked = fakeLastChecked1; sandbox.stub(Math, 'random').returns(0); @@ -105,19 +105,15 @@ describe('Cache Class', () => { assert.strictEqual( cache.shouldRefresh(customDuration), false, - 'Cache should not refresh within custom duration' + 'Cache should not refresh within custom duration', ); // Elapsed time = 7 hours - const fakeLastChecked2 = fakeNow - (7 * 60 * 60 * 1000); + const fakeLastChecked2 = fakeNow - 7 * 60 * 60 * 1000; cache.lastChecked = fakeLastChecked2; // 7 hours > 6 hours => shouldRefresh = true - assert.strictEqual( - cache.shouldRefresh(customDuration), - true, - 'Cache should refresh after custom duration' - ); + assert.strictEqual(cache.shouldRefresh(customDuration), true, 'Cache should refresh after custom duration'); }); }); diff --git a/tests/unit/encryption/kms.js b/tests/unit/encryption/kms.js index ff94a77d8a..98ff7d4fd2 100644 --- a/tests/unit/encryption/kms.js +++ b/tests/unit/encryption/kms.js @@ -6,8 +6,7 @@ const Common = require('../../../lib/kms/common'); const { cleanup, DummyRequestLogger } = require('../helpers'); const log = new DummyRequestLogger(); -const dummyBucket = new BucketInfo( - 'dummyBucket', 'dummyOwnerId', 'Joe, John', new Date().toJSON()); +const dummyBucket = new BucketInfo('dummyBucket', 'dummyOwnerId', 'Joe, John', new Date().toJSON()); describe('KMS unit tests', () => { beforeEach(() => { @@ -20,17 +19,15 @@ describe('KMS unit tests', () => { 'x-amz-scal-server-side-encryption': algorithm, }; const sseConfig = parseBucketEncryptionHeaders(headers); - KMS.bucketLevelEncryption( - dummyBucket, sseConfig, log, - (err, sseInfo) => { - assert.strictEqual(err, null); - assert.strictEqual(sseInfo.cryptoScheme, 1); - assert.strictEqual(sseInfo.mandatory, true); - assert.strictEqual(sseInfo.algorithm, algorithm); - assert.notEqual(sseInfo.masterKeyId, undefined); - assert.notEqual(sseInfo.masterKeyId, null); - done(); - }); + KMS.bucketLevelEncryption(dummyBucket, sseConfig, log, (err, sseInfo) => { + assert.strictEqual(err, null); + assert.strictEqual(sseInfo.cryptoScheme, 1); + assert.strictEqual(sseInfo.mandatory, true); + assert.strictEqual(sseInfo.algorithm, algorithm); + assert.notEqual(sseInfo.masterKeyId, undefined); + assert.notEqual(sseInfo.masterKeyId, null); + done(); + }); }); it('should construct a sse info object on aws:kms', done => { @@ -41,49 +38,48 @@ describe('KMS unit tests', () => { 'x-amz-scal-server-side-encryption-aws-kms-key-id': masterKeyId, }; const sseConfig = parseBucketEncryptionHeaders(headers); - KMS.bucketLevelEncryption( - dummyBucket, sseConfig, log, - (err, sseInfo) => { - assert.strictEqual(err, null); - assert.strictEqual(sseInfo.cryptoScheme, 1); - assert.strictEqual(sseInfo.mandatory, true); - assert.strictEqual(sseInfo.algorithm, 'aws:kms'); - assert.strictEqual(sseInfo.configuredMasterKeyId, `${KMS.arnPrefix}${masterKeyId}`); - done(); - }); + KMS.bucketLevelEncryption(dummyBucket, sseConfig, log, (err, sseInfo) => { + assert.strictEqual(err, null); + assert.strictEqual(sseInfo.cryptoScheme, 1); + assert.strictEqual(sseInfo.mandatory, true); + assert.strictEqual(sseInfo.algorithm, 'aws:kms'); + assert.strictEqual(sseInfo.configuredMasterKeyId, `${KMS.arnPrefix}${masterKeyId}`); + done(); + }); }); - it('should not construct a sse info object if ' + - 'x-amz-scal-server-side-encryption header contains invalid ' + - 'algorithm option', done => { - const algorithm = 'garbage'; - const masterKeyId = 'foobarbaz'; - const headers = { - 'x-amz-scal-server-side-encryption': algorithm, - 'x-amz-scal-server-side-encryption-aws-kms-key-id': masterKeyId, - }; - const sseConfig = parseBucketEncryptionHeaders(headers); - KMS.bucketLevelEncryption( - dummyBucket, sseConfig, log, - (err, sseInfo) => { + it( + 'should not construct a sse info object if ' + + 'x-amz-scal-server-side-encryption header contains invalid ' + + 'algorithm option', + done => { + const algorithm = 'garbage'; + const masterKeyId = 'foobarbaz'; + const headers = { + 'x-amz-scal-server-side-encryption': algorithm, + 'x-amz-scal-server-side-encryption-aws-kms-key-id': masterKeyId, + }; + const sseConfig = parseBucketEncryptionHeaders(headers); + KMS.bucketLevelEncryption(dummyBucket, sseConfig, log, (err, sseInfo) => { assert.strictEqual(err, null); assert.strictEqual(sseInfo, null); done(); }); - }); + }, + ); - it('should not construct a sse info object if no ' + - 'x-amz-scal-server-side-encryption header included with request', + it( + 'should not construct a sse info object if no ' + + 'x-amz-scal-server-side-encryption header included with request', done => { const sseConfig = parseBucketEncryptionHeaders({}); - KMS.bucketLevelEncryption( - dummyBucket, sseConfig, log, - (err, sseInfo) => { - assert.strictEqual(err, null); - assert.strictEqual(sseInfo, null); - done(); - }); - }); + KMS.bucketLevelEncryption(dummyBucket, sseConfig, log, (err, sseInfo) => { + assert.strictEqual(err, null); + assert.strictEqual(sseInfo, null); + done(); + }); + }, + ); it('should create a cipher bundle for AES256', done => { const algorithm = 'AES256'; @@ -91,22 +87,16 @@ describe('KMS unit tests', () => { 'x-amz-scal-server-side-encryption': algorithm, }; const sseConfig = parseBucketEncryptionHeaders(headers); - KMS.bucketLevelEncryption( - dummyBucket, sseConfig, log, - (err, sseInfo) => { - KMS.createCipherBundle( - sseInfo, log, (err, cipherBundle) => { - assert.strictEqual(cipherBundle.algorithm, - sseInfo.algorithm); - assert.strictEqual(cipherBundle.masterKeyId, - sseInfo.masterKeyId); - assert.strictEqual(cipherBundle.cryptoScheme, - sseInfo.cryptoScheme); - assert.notEqual(cipherBundle.cipheredDataKey, null); - assert.notEqual(cipherBundle.cipher, null); - done(); - }); + KMS.bucketLevelEncryption(dummyBucket, sseConfig, log, (err, sseInfo) => { + KMS.createCipherBundle(sseInfo, log, (err, cipherBundle) => { + assert.strictEqual(cipherBundle.algorithm, sseInfo.algorithm); + assert.strictEqual(cipherBundle.masterKeyId, sseInfo.masterKeyId); + assert.strictEqual(cipherBundle.cryptoScheme, sseInfo.cryptoScheme); + assert.notEqual(cipherBundle.cipheredDataKey, null); + assert.notEqual(cipherBundle.cipher, null); + done(); }); + }); }); it('should create a cipher bundle for aws:kms', done => { @@ -115,33 +105,24 @@ describe('KMS unit tests', () => { }; let masterKeyId; let sseConfig = parseBucketEncryptionHeaders(headers); - KMS.bucketLevelEncryption( - dummyBucket, sseConfig, log, - (err, sseInfo) => { - assert.strictEqual(err, null); - masterKeyId = sseInfo.bucketKeyId; - }); + KMS.bucketLevelEncryption(dummyBucket, sseConfig, log, (err, sseInfo) => { + assert.strictEqual(err, null); + masterKeyId = sseInfo.bucketKeyId; + }); headers['x-amz-scal-server-side-encryption'] = 'aws:kms'; - headers['x-amz-scal-server-side-encryption-aws-kms-key-id'] = - masterKeyId; + headers['x-amz-scal-server-side-encryption-aws-kms-key-id'] = masterKeyId; sseConfig = parseBucketEncryptionHeaders(headers); - KMS.bucketLevelEncryption( - dummyBucket, sseConfig, log, - (err, sseInfo) => { - KMS.createCipherBundle( - sseInfo, log, (err, cipherBundle) => { - assert.strictEqual(cipherBundle.algorithm, - sseInfo.algorithm); - assert.strictEqual(cipherBundle.masterKeyId, - sseInfo.masterKeyId); - assert.strictEqual(cipherBundle.cryptoScheme, - sseInfo.cryptoScheme); - assert.notEqual(cipherBundle.cipheredDataKey, null); - assert.notEqual(cipherBundle.cipher, null); - done(); - }); + KMS.bucketLevelEncryption(dummyBucket, sseConfig, log, (err, sseInfo) => { + KMS.createCipherBundle(sseInfo, log, (err, cipherBundle) => { + assert.strictEqual(cipherBundle.algorithm, sseInfo.algorithm); + assert.strictEqual(cipherBundle.masterKeyId, sseInfo.masterKeyId); + assert.strictEqual(cipherBundle.cryptoScheme, sseInfo.cryptoScheme); + assert.notEqual(cipherBundle.cipheredDataKey, null); + assert.notEqual(cipherBundle.cipher, null); + done(); }); + }); }); /* cb(err, cipherBundle, decipherBundle) */ @@ -151,37 +132,30 @@ describe('KMS unit tests', () => { 'x-amz-scal-server-side-encryption': algorithm, }; const sseConfig = parseBucketEncryptionHeaders(headers); - KMS.bucketLevelEncryption( - dummyBucket, sseConfig, log, - (err, sseInfo) => { + KMS.bucketLevelEncryption(dummyBucket, sseConfig, log, (err, sseInfo) => { + if (err) { + cb(err); + return; + } + KMS.createCipherBundle(sseInfo, log, (err, cipherBundle) => { if (err) { cb(err); return; } - KMS.createCipherBundle( - sseInfo, log, (err, cipherBundle) => { - if (err) { - cb(err); - return; - } - const creatingSseInfo = sseInfo; - creatingSseInfo.cipheredDataKey = - Buffer.from(cipherBundle.cipheredDataKey, 'base64'); - KMS.createDecipherBundle( - sseInfo, 0, log, (err, decipherBundle) => { - if (err) { - cb(err); - return; - } - assert.strictEqual(typeof decipherBundle, - 'object'); - assert.strictEqual(decipherBundle.cryptoScheme, - cipherBundle.cryptoScheme); - assert.notEqual(decipherBundle.decipher, null); - cb(null, cipherBundle, decipherBundle); - }); - }); + const creatingSseInfo = sseInfo; + creatingSseInfo.cipheredDataKey = Buffer.from(cipherBundle.cipheredDataKey, 'base64'); + KMS.createDecipherBundle(sseInfo, 0, log, (err, decipherBundle) => { + if (err) { + cb(err); + return; + } + assert.strictEqual(typeof decipherBundle, 'object'); + assert.strictEqual(decipherBundle.cryptoScheme, cipherBundle.cryptoScheme); + assert.notEqual(decipherBundle.decipher, null); + cb(null, cipherBundle, decipherBundle); + }); }); + }); } it('should cipher and decipher a datastream', done => { @@ -199,8 +173,7 @@ describe('KMS unit tests', () => { }); }); - it('should increment the IV by modifying the last two positions of ' + - 'the buffer', () => { + it('should increment the IV by modifying the last two positions of ' + 'the buffer', () => { const derivedIV = Buffer.from('aaaaaaff', 'hex'); const counter = 6; const incrementedIV = Common._incrementIV(derivedIV, counter); @@ -208,8 +181,7 @@ describe('KMS unit tests', () => { assert.deepStrictEqual(incrementedIV, expected); }); - it('should increment the IV by incrementing the last position of the ' + - 'buffer', () => { + it('should increment the IV by incrementing the last position of the ' + 'buffer', () => { const derivedIV = Buffer.from('aaaaaaf0', 'hex'); const counter = 6; const incrementedIV = Common._incrementIV(derivedIV, counter); @@ -217,8 +189,7 @@ describe('KMS unit tests', () => { assert.deepStrictEqual(incrementedIV, expected); }); - it('should increment the IV by shifting each position in the ' + - 'buffer', () => { + it('should increment the IV by shifting each position in the ' + 'buffer', () => { const derivedIV = Buffer.from('ffffffff', 'hex'); const counter = 1; const incrementedIV = Common._incrementIV(derivedIV, counter); diff --git a/tests/unit/githubScripts/asyncMigrationScripts.js b/tests/unit/githubScripts/asyncMigrationScripts.js index 40ce384e13..aa2e65ad7f 100644 --- a/tests/unit/githubScripts/asyncMigrationScripts.js +++ b/tests/unit/githubScripts/asyncMigrationScripts.js @@ -72,12 +72,10 @@ describe('CI async migration scripts', () => { tempDirs.push(dir); fs.mkdirSync(path.join(dir, 'lib'), { recursive: true }); - fs.writeFileSync(path.join(dir, 'lib/newFile.js'), [ - 'function badStyle(param, cb) {', - ' return cb(null, param);', - '}', - '', - ].join('\n')); + fs.writeFileSync( + path.join(dir, 'lib/newFile.js'), + ['function badStyle(param, cb) {', ' return cb(null, param);', '}', ''].join('\n'), + ); run('git', ['add', 'lib/newFile.js'], dir); const result = runNodeScript(checkDiffScript, dir); diff --git a/tests/unit/healthchecks/clientCheck.js b/tests/unit/healthchecks/clientCheck.js index 2e514af8b4..f636516cf7 100644 --- a/tests/unit/healthchecks/clientCheck.js +++ b/tests/unit/healthchecks/clientCheck.js @@ -30,19 +30,27 @@ describe('clientCheck - failure detection logic', () => { }); it('should succeed when all backends are healthy', done => { - dataStub.callsFake((log, cb) => cb(null, { - 'sproxyd-loc1': { code: 200, message: 'OK' }, - 'sproxyd-loc2': { code: 200, message: 'OK' }, - })); - metadataStub.callsFake((log, cb) => cb(null, { - metadata: { code: 200, message: 'OK' }, - })); - vaultStub.callsFake((log, cb) => cb(null, { - vault: { code: 200, message: 'OK' }, - })); - kmsStub.callsFake((log, cb) => cb(null, { - kms: { code: 200, message: 'OK' }, - })); + dataStub.callsFake((log, cb) => + cb(null, { + 'sproxyd-loc1': { code: 200, message: 'OK' }, + 'sproxyd-loc2': { code: 200, message: 'OK' }, + }), + ); + metadataStub.callsFake((log, cb) => + cb(null, { + metadata: { code: 200, message: 'OK' }, + }), + ); + vaultStub.callsFake((log, cb) => + cb(null, { + vault: { code: 200, message: 'OK' }, + }), + ); + kmsStub.callsFake((log, cb) => + cb(null, { + kms: { code: 200, message: 'OK' }, + }), + ); clientCheck(false, log, (err, result) => { assert.ifError(err); @@ -58,19 +66,27 @@ describe('clientCheck - failure detection logic', () => { }); it('should fail when ALL backends of data client fail while metadata is healthy', done => { - dataStub.callsFake((log, cb) => cb(null, { - 'sproxyd-loc1': { error: errors.InternalError, code: 500 }, - 'sproxyd-loc2': { error: errors.InternalError, code: 500 }, - })); - metadataStub.callsFake((log, cb) => cb(null, { - metadata: { code: 200, message: 'OK' }, - })); - vaultStub.callsFake((log, cb) => cb(null, { - vault: { code: 200, message: 'OK' }, - })); - kmsStub.callsFake((log, cb) => cb(null, { - kms: { code: 200, message: 'OK' }, - })); + dataStub.callsFake((log, cb) => + cb(null, { + 'sproxyd-loc1': { error: errors.InternalError, code: 500 }, + 'sproxyd-loc2': { error: errors.InternalError, code: 500 }, + }), + ); + metadataStub.callsFake((log, cb) => + cb(null, { + metadata: { code: 200, message: 'OK' }, + }), + ); + vaultStub.callsFake((log, cb) => + cb(null, { + vault: { code: 200, message: 'OK' }, + }), + ); + kmsStub.callsFake((log, cb) => + cb(null, { + kms: { code: 200, message: 'OK' }, + }), + ); clientCheck(false, log, (err, result) => { assert(err); @@ -87,19 +103,27 @@ describe('clientCheck - failure detection logic', () => { }); it('should succeed when ONE data location fails but another is healthy', done => { - dataStub.callsFake((log, cb) => cb(null, { - 'sproxyd-loc1': { error: errors.InternalError, code: 500 }, - 'sproxyd-loc2': { code: 200, message: 'OK' }, - })); - metadataStub.callsFake((log, cb) => cb(null, { - metadata: { code: 200, message: 'OK' }, - })); - vaultStub.callsFake((log, cb) => cb(null, { - vault: { code: 200, message: 'OK' }, - })); - kmsStub.callsFake((log, cb) => cb(null, { - kms: { code: 200, message: 'OK' }, - })); + dataStub.callsFake((log, cb) => + cb(null, { + 'sproxyd-loc1': { error: errors.InternalError, code: 500 }, + 'sproxyd-loc2': { code: 200, message: 'OK' }, + }), + ); + metadataStub.callsFake((log, cb) => + cb(null, { + metadata: { code: 200, message: 'OK' }, + }), + ); + vaultStub.callsFake((log, cb) => + cb(null, { + vault: { code: 200, message: 'OK' }, + }), + ); + kmsStub.callsFake((log, cb) => + cb(null, { + kms: { code: 200, message: 'OK' }, + }), + ); clientCheck(false, log, (err, result) => { assert.ifError(err); @@ -115,19 +139,27 @@ describe('clientCheck - failure detection logic', () => { }); it('should fail when ALL backends of multiple clients fail', done => { - dataStub.callsFake((log, cb) => cb(null, { - 'sproxyd-loc1': { error: errors.InternalError, code: 500 }, - 'sproxyd-loc2': { error: errors.InternalError, code: 500 }, - })); - metadataStub.callsFake((log, cb) => cb(null, { - metadata: { error: errors.InternalError, code: 500 }, - })); - vaultStub.callsFake((log, cb) => cb(null, { - vault: { code: 200, message: 'OK' }, - })); - kmsStub.callsFake((log, cb) => cb(null, { - kms: { code: 200, message: 'OK' }, - })); + dataStub.callsFake((log, cb) => + cb(null, { + 'sproxyd-loc1': { error: errors.InternalError, code: 500 }, + 'sproxyd-loc2': { error: errors.InternalError, code: 500 }, + }), + ); + metadataStub.callsFake((log, cb) => + cb(null, { + metadata: { error: errors.InternalError, code: 500 }, + }), + ); + vaultStub.callsFake((log, cb) => + cb(null, { + vault: { code: 200, message: 'OK' }, + }), + ); + kmsStub.callsFake((log, cb) => + cb(null, { + kms: { code: 200, message: 'OK' }, + }), + ); clientCheck(false, log, (err, result) => { assert(err); @@ -145,15 +177,21 @@ describe('clientCheck - failure detection logic', () => { it('should succeed when client returns empty result', done => { dataStub.callsFake((log, cb) => cb(null, {})); - metadataStub.callsFake((log, cb) => cb(null, { - metadata: { code: 200, message: 'OK' }, - })); - vaultStub.callsFake((log, cb) => cb(null, { - vault: { code: 200, message: 'OK' }, - })); - kmsStub.callsFake((log, cb) => cb(null, { - kms: { code: 200, message: 'OK' }, - })); + metadataStub.callsFake((log, cb) => + cb(null, { + metadata: { code: 200, message: 'OK' }, + }), + ); + vaultStub.callsFake((log, cb) => + cb(null, { + vault: { code: 200, message: 'OK' }, + }), + ); + kmsStub.callsFake((log, cb) => + cb(null, { + kms: { code: 200, message: 'OK' }, + }), + ); clientCheck(false, log, (err, result) => { assert.ifError(err); @@ -167,34 +205,44 @@ describe('clientCheck - failure detection logic', () => { }); describe('external backend error handling', () => { - it('should NOT fail on external backend errors during normal operation ' + - '(flightCheckOnStartUp=false)', done => { - dataStub.callsFake((log, cb) => cb(null, { - 's3-backend': { error: errors.InternalError, code: 500, external: true }, - })); - metadataStub.callsFake((log, cb) => cb(null, { - metadata: { code: 200, message: 'OK' }, - })); - vaultStub.callsFake((log, cb) => cb(null, {})); - kmsStub.callsFake((log, cb) => cb(null, {})); + it( + 'should NOT fail on external backend errors during normal operation ' + '(flightCheckOnStartUp=false)', + done => { + dataStub.callsFake((log, cb) => + cb(null, { + 's3-backend': { error: errors.InternalError, code: 500, external: true }, + }), + ); + metadataStub.callsFake((log, cb) => + cb(null, { + metadata: { code: 200, message: 'OK' }, + }), + ); + vaultStub.callsFake((log, cb) => cb(null, {})); + kmsStub.callsFake((log, cb) => cb(null, {})); - clientCheck(false, log, (err, result) => { - assert.ifError(err); - assert.deepStrictEqual(result, { - 's3-backend': { error: errors.InternalError, code: 500, external: true }, - metadata: { code: 200, message: 'OK' }, + clientCheck(false, log, (err, result) => { + assert.ifError(err); + assert.deepStrictEqual(result, { + 's3-backend': { error: errors.InternalError, code: 500, external: true }, + metadata: { code: 200, message: 'OK' }, + }); + done(); }); - done(); - }); - }); + }, + ); it('should fail on external backend errors during startup (flightCheckOnStartUp=true)', done => { - dataStub.callsFake((log, cb) => cb(null, { - 's3-backend': { error: errors.InternalError, code: 500, external: true }, - })); - metadataStub.callsFake((log, cb) => cb(null, { - metadata: { code: 200, message: 'OK' }, - })); + dataStub.callsFake((log, cb) => + cb(null, { + 's3-backend': { error: errors.InternalError, code: 500, external: true }, + }), + ); + metadataStub.callsFake((log, cb) => + cb(null, { + metadata: { code: 200, message: 'OK' }, + }), + ); vaultStub.callsFake((log, cb) => cb(null, {})); kmsStub.callsFake((log, cb) => cb(null, {})); @@ -209,27 +257,34 @@ describe('clientCheck - failure detection logic', () => { }); }); - it('should succeed when external backend fails but internal backend is healthy ' + - '(flightCheckOnStartUp=false)', done => { - dataStub.callsFake((log, cb) => cb(null, { - 'sproxyd-loc1': { code: 200, message: 'OK' }, - 's3-backend': { error: errors.InternalError, code: 500, external: true }, - })); - metadataStub.callsFake((log, cb) => cb(null, { - metadata: { code: 200, message: 'OK' }, - })); - vaultStub.callsFake((log, cb) => cb(null, {})); - kmsStub.callsFake((log, cb) => cb(null, {})); + it( + 'should succeed when external backend fails but internal backend is healthy ' + + '(flightCheckOnStartUp=false)', + done => { + dataStub.callsFake((log, cb) => + cb(null, { + 'sproxyd-loc1': { code: 200, message: 'OK' }, + 's3-backend': { error: errors.InternalError, code: 500, external: true }, + }), + ); + metadataStub.callsFake((log, cb) => + cb(null, { + metadata: { code: 200, message: 'OK' }, + }), + ); + vaultStub.callsFake((log, cb) => cb(null, {})); + kmsStub.callsFake((log, cb) => cb(null, {})); - clientCheck(false, log, (err, result) => { - assert.ifError(err); - assert.deepStrictEqual(result, { - 'sproxyd-loc1': { code: 200, message: 'OK' }, - 's3-backend': { error: errors.InternalError, code: 500, external: true }, - metadata: { code: 200, message: 'OK' }, + clientCheck(false, log, (err, result) => { + assert.ifError(err); + assert.deepStrictEqual(result, { + 'sproxyd-loc1': { code: 200, message: 'OK' }, + 's3-backend': { error: errors.InternalError, code: 500, external: true }, + metadata: { code: 200, message: 'OK' }, + }); + done(); }); - done(); - }); - }); + }, + ); }); }); diff --git a/tests/unit/helpers.js b/tests/unit/helpers.js index a8f1f594c4..8075da8df5 100644 --- a/tests/unit/helpers.js +++ b/tests/unit/helpers.js @@ -34,11 +34,9 @@ const testsRangeOnEmptyFile = [ function makeid(size) { let text = ''; - const possible = - 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'; + const possible = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'; for (let i = 0; i < size; i += 1) { - text += possible - .charAt(Math.floor(Math.random() * possible.length)); + text += possible.charAt(Math.floor(Math.random() * possible.length)); } return text; } @@ -62,16 +60,14 @@ function timeDiff(startTime) { const timeArray = process.hrtime(startTime); // timeArray[0] is whole seconds // timeArray[1] is remaining nanoseconds - const milliseconds = (timeArray[0] * 1000) + (timeArray[1] / 1e6); + const milliseconds = timeArray[0] * 1000 + timeArray[1] / 1e6; return milliseconds; } function makeAuthInfo(accessKey, userName) { const canIdMap = { - accessKey1: '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7' - + 'cd47ef2be', - accessKey2: '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7' - + 'cd47ef2bf', + accessKey1: '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7' + 'cd47ef2be', + accessKey2: '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7' + 'cd47ef2bf', lifecycleKey1: '0123456789abcdef/lifecycle', default: crypto.randomBytes(32).toString('hex'), }; @@ -139,32 +135,23 @@ class WebsiteConfig { }); } - xml.push(''); + xml.push(''); if (this.IndexDocument) { - xml.push('', - `${this.IndexDocument.Suffix}`, - ''); + xml.push('', `${this.IndexDocument.Suffix}`, ''); } if (this.ErrorDocument) { - xml.push('', - `${this.ErrorDocument.Key}`, - ''); + xml.push('', `${this.ErrorDocument.Key}`, ''); } if (this.RedirectAllRequestsTo) { xml.push(''); if (this.RedirectAllRequestsTo.HostName) { - xml.push('', - `${this.RedirectAllRequestsTo.HostName})`, - ''); + xml.push('', `${this.RedirectAllRequestsTo.HostName})`, ''); } if (this.RedirectAllRequestsTo.Protocol) { - xml.push('', - `${this.RedirectAllRequestsTo.Protocol})`, - ''); + xml.push('', `${this.RedirectAllRequestsTo.Protocol})`, ''); } xml.push(''); } @@ -193,8 +180,7 @@ class WebsiteConfig { } } -function createAlteredRequest(alteredItems, objToAlter, - baseOuterObj, baseInnerObj) { +function createAlteredRequest(alteredItems, objToAlter, baseOuterObj, baseInnerObj) { const alteredRequest = Object.assign({}, baseOuterObj); const alteredNestedObj = Object.assign({}, baseInnerObj); Object.keys(alteredItems).forEach(key => { @@ -205,8 +191,8 @@ function createAlteredRequest(alteredItems, objToAlter, } function cleanup() { - metadata.buckets = new Map; - metadata.keyMaps = new Map; + metadata.buckets = new Map(); + metadata.keyMaps = new Map(); // Set data store array back to empty array ds.length = 0; // Set data store key count back to 1 @@ -276,19 +262,22 @@ class DummyRequestLogger { class CorsConfigTester { constructor(params) { - this._cors = [{ - allowedMethods: ['PUT', 'POST', 'DELETE'], - allowedOrigins: ['http://www.example.com'], - allowedHeaders: ['*'], - maxAgeSeconds: 3000, - exposeHeaders: ['x-amz-server-side-encryption'], - }, { - id: 'testid', - allowedMethods: ['GET'], - allowedOrigins: ['*'], - allowedHeaders: ['*'], - maxAgeSeconds: 3000, - }]; + this._cors = [ + { + allowedMethods: ['PUT', 'POST', 'DELETE'], + allowedOrigins: ['http://www.example.com'], + allowedHeaders: ['*'], + maxAgeSeconds: 3000, + exposeHeaders: ['x-amz-server-side-encryption'], + }, + { + id: 'testid', + allowedMethods: ['GET'], + allowedOrigins: ['*'], + allowedHeaders: ['*'], + maxAgeSeconds: 3000, + }, + ]; if (params) { Object.keys(params).forEach(key => { @@ -306,14 +295,12 @@ class CorsConfigTester { xml.push(''); this._cors.forEach(rule => { xml.push(''); - ['allowedMethods', 'allowedOrigins', 'allowedHeaders', - 'exposeHeaders', 'maxAgeSeconds'] - .forEach(key => { + ['allowedMethods', 'allowedOrigins', 'allowedHeaders', 'exposeHeaders', 'maxAgeSeconds'].forEach(key => { if (rule[key] && Array.isArray(rule[key])) { - const element = key === 'maxAgeSeconds' ? - key.charAt(0).toUpperCase() + key.slice(1) : - key.charAt(0).toUpperCase() + - key.slice(1, -1); + const element = + key === 'maxAgeSeconds' + ? key.charAt(0).toUpperCase() + key.slice(1) + : key.charAt(0).toUpperCase() + key.slice(1, -1); rule[key].forEach(value => { xml.push(`<${element}>${value}`); }); @@ -323,8 +310,7 @@ class CorsConfigTester { xml.push(`${rule.id}`); } if (rule.maxAgeSeconds && !Array.isArray(rule.maxAgeSeconds)) { - xml.push(`${rule.maxAgeSeconds}` + - ''); + xml.push(`${rule.maxAgeSeconds}` + ''); } xml.push(''); }); @@ -344,8 +330,7 @@ class CorsConfigTester { }; if (method === 'PUT') { request.post = body || this.constructXml(); - request.headers['content-md5'] = crypto.createHash('md5') - .update(request.post, 'utf8').digest('base64'); + request.headers['content-md5'] = crypto.createHash('md5').update(request.post, 'utf8').digest('base64'); } return request; } @@ -384,10 +369,11 @@ const versioningTestUtils = { query: { versioning: '' }, actionImplicitDenies: false, }; - const xml = '' + - `${status}` + - ''; + const xml = + '' + + `${status}` + + ''; request.post = xml; return request; }, @@ -414,8 +400,7 @@ class TaggingConfigTester { constructXml() { const xml = []; - xml.push('' + - ' '); + xml.push('' + ' '); Object.keys(this._tags).forEach(key => { const value = this._tags[key]; xml.push(`${key}${value}`); @@ -437,8 +422,7 @@ class TaggingConfigTester { }; if (method === 'PUT') { request.post = body || this.constructXml(); - request.headers['content-md5'] = crypto.createHash('md5') - .update(request.post, 'utf8').digest('base64'); + request.headers['content-md5'] = crypto.createHash('md5').update(request.post, 'utf8').digest('base64'); } return request; } @@ -455,8 +439,7 @@ class TaggingConfigTester { }; if (method === 'PUT') { request.post = body || this.constructXml(); - request.headers['content-md5'] = crypto.createHash('md5') - .update(request.post, 'utf8').digest('base64'); + request.headers['content-md5'] = crypto.createHash('md5').update(request.post, 'utf8').digest('base64'); } return request; } @@ -499,16 +482,13 @@ class AccessControlPolicy { } }); } - xml.push('', ''); + xml.push('', ''); _pushChildren(this.Owner); xml.push('', ''); this.AccessControlList.forEach(grant => { xml.push('', ``); _pushChildren(grant.Grantee); - xml.push('', - `${grant.Permission}`, - ''); + xml.push('', `${grant.Permission}`, ''); }); xml.push('', ''); return xml.join(''); @@ -516,9 +496,16 @@ class AccessControlPolicy { } function createRequestContext(apiMethod, request) { - return new RequestContext(request.headers, - request.query, request.bucketName, request.objectKey, - '127.0.0.1', false, apiMethod, 's3'); + return new RequestContext( + request.headers, + request.query, + request.bucketName, + request.objectKey, + '127.0.0.1', + false, + apiMethod, + 's3', + ); } module.exports = { diff --git a/tests/unit/internal/routeVeeam.js b/tests/unit/internal/routeVeeam.js index 49f2d3ba4b..21c9f62e19 100644 --- a/tests/unit/internal/routeVeeam.js +++ b/tests/unit/internal/routeVeeam.js @@ -44,21 +44,19 @@ describe('RouteVeeam: checkBucketAndKey', () => { }); }); - [ - ['test', 'badObjectKey', null, 'GET', log], - ].forEach(test => { + [['test', 'badObjectKey', null, 'GET', log]].forEach(test => { it(`should return InvalidArgument for "${test[1]}" object name`, () => { assert.strictEqual(routeVeeam.checkBucketAndKey(...test).is.InvalidArgument, true); }); }); - [ - ['test', '.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/system.xml', { random: 'queryparam' }, 'GET', log], - ].forEach(test => { - it(`should return InvalidRequest for "${test[1]}" object name`, () => { - assert.strictEqual(routeVeeam.checkBucketAndKey(...test).is.InvalidRequest, true); - }); - }); + [['test', '.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/system.xml', { random: 'queryparam' }, 'GET', log]].forEach( + test => { + it(`should return InvalidRequest for "${test[1]}" object name`, () => { + assert.strictEqual(routeVeeam.checkBucketAndKey(...test).is.InvalidRequest, true); + }); + }, + ); [ ['test', '.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/system.xml', null, 'GET', log], @@ -234,14 +232,19 @@ describe('RouteVeeam: routeVeeam', () => { url: '/bucket/veeam', }); req.method = 'PATCH'; - routeVeeam.routeVeeam('127.0.0.1', req, { - setHeader: () => {}, - writeHead: () => {}, - end: data => { - assert(data.includes('MethodNotAllowed')); - done(); + routeVeeam.routeVeeam( + '127.0.0.1', + req, + { + setHeader: () => {}, + writeHead: () => {}, + end: data => { + assert(data.includes('MethodNotAllowed')); + done(); + }, + headersSent: false, }, - headersSent: false, - }, log); + log, + ); }); }); diff --git a/tests/unit/internal/veeam/schemas/system.js b/tests/unit/internal/veeam/schemas/system.js index ee92bfbf4c..06fa21ea9e 100644 --- a/tests/unit/internal/veeam/schemas/system.js +++ b/tests/unit/internal/veeam/schemas/system.js @@ -18,7 +18,7 @@ describe('RouteVeeam: validateSystemSchema 1.0', () => { CapacityInfo: true, UploadSessions: true, IAMSTS: true, - } + }, }, SystemRecommendations: { S3ConcurrentTaskLimit: 0, @@ -89,7 +89,6 @@ describe('RouteVeeam: validateSystemSchema 1.0', () => { }); }); - describe('RouteVeeam: validateSystemSchema unknown version', () => { const protocolVersion = '"1.1"'; [ @@ -101,7 +100,7 @@ describe('RouteVeeam: validateSystemSchema unknown version', () => { CapacityInfo: true, UploadSessions: true, IAMSTS: true, - } + }, }, SystemRecommendations: { S3ConcurrentTaskLimit: 0, diff --git a/tests/unit/management/agent.js b/tests/unit/management/agent.js index da806a7521..57c7c7c1a4 100644 --- a/tests/unit/management/agent.js +++ b/tests/unit/management/agent.js @@ -1,8 +1,6 @@ const assert = require('assert'); -const { - createWSAgent, -} = require('../../../lib/management/push'); +const { createWSAgent } = require('../../../lib/management/push'); const proxy = 'http://proxy:3128/'; const logger = { info: () => {} }; @@ -10,57 +8,93 @@ const logger = { info: () => {} }; function testVariableSet(httpProxy, httpsProxy, allProxy, noProxy) { return () => { it(`should use ${httpProxy} environment variable`, () => { - let agent = createWSAgent('https://pushserver', { - [httpProxy]: 'http://proxy:3128', - }, logger); + let agent = createWSAgent( + 'https://pushserver', + { + [httpProxy]: 'http://proxy:3128', + }, + logger, + ); assert.equal(agent, null); - agent = createWSAgent('http://pushserver', { - [httpProxy]: proxy, - }, logger); + agent = createWSAgent( + 'http://pushserver', + { + [httpProxy]: proxy, + }, + logger, + ); assert.equal(agent.proxy.href, proxy); }); it(`should use ${httpsProxy} environment variable`, () => { - let agent = createWSAgent('http://pushserver', { - [httpsProxy]: proxy, - }, logger); + let agent = createWSAgent( + 'http://pushserver', + { + [httpsProxy]: proxy, + }, + logger, + ); assert.equal(agent, null); - agent = createWSAgent('https://pushserver', { - [httpsProxy]: proxy, - }, logger); + agent = createWSAgent( + 'https://pushserver', + { + [httpsProxy]: proxy, + }, + logger, + ); assert.equal(agent.proxy.href, proxy); }); it(`should use ${allProxy} environment variable`, () => { - let agent = createWSAgent('http://pushserver', { - [allProxy]: proxy, - }, logger); + let agent = createWSAgent( + 'http://pushserver', + { + [allProxy]: proxy, + }, + logger, + ); assert.equal(agent.proxy.href, proxy); - agent = createWSAgent('https://pushserver', { - [allProxy]: proxy, - }, logger); + agent = createWSAgent( + 'https://pushserver', + { + [allProxy]: proxy, + }, + logger, + ); assert.equal(agent.proxy.href, proxy); }); it(`should use ${noProxy} environment variable`, () => { - let agent = createWSAgent('http://pushserver', { - [noProxy]: 'pushserver', - }, logger); + let agent = createWSAgent( + 'http://pushserver', + { + [noProxy]: 'pushserver', + }, + logger, + ); assert.equal(agent, null); - agent = createWSAgent('http://pushserver', { - [noProxy]: 'pushserver', - [httpProxy]: proxy, - }, logger); + agent = createWSAgent( + 'http://pushserver', + { + [noProxy]: 'pushserver', + [httpProxy]: proxy, + }, + logger, + ); assert.equal(agent, null); - agent = createWSAgent('http://pushserver', { - [noProxy]: 'pushserver2', - [httpProxy]: proxy, - }, logger); + agent = createWSAgent( + 'http://pushserver', + { + [noProxy]: 'pushserver2', + [httpProxy]: proxy, + }, + logger, + ); assert.equal(agent.proxy.href, proxy); }); }; @@ -74,9 +108,7 @@ describe('Websocket connection agent', () => { }); }); - describe('with lowercase proxy env', - testVariableSet('http_proxy', 'https_proxy', 'all_proxy', 'no_proxy')); + describe('with lowercase proxy env', testVariableSet('http_proxy', 'https_proxy', 'all_proxy', 'no_proxy')); - describe('with uppercase proxy env', - testVariableSet('HTTP_PROXY', 'HTTPS_PROXY', 'ALL_PROXY', 'NO_PROXY')); + describe('with uppercase proxy env', testVariableSet('HTTP_PROXY', 'HTTPS_PROXY', 'ALL_PROXY', 'NO_PROXY')); }); diff --git a/tests/unit/management/configuration.js b/tests/unit/management/configuration.js index 71134c4f3c..ef38fcd435 100644 --- a/tests/unit/management/configuration.js +++ b/tests/unit/management/configuration.js @@ -8,26 +8,20 @@ const metadata = require('../../../lib/metadata/wrapper'); const managementDatabaseName = 'PENSIEVE'; const tokenConfigurationKey = 'auth/zenko/remote-management-token'; -const { privateKey, accessKey, decryptedSecretKey, secretKey, canonicalId, - userName } = require('./resources.json'); +const { privateKey, accessKey, decryptedSecretKey, secretKey, canonicalId, userName } = require('./resources.json'); const shortid = '123456789012'; const email = 'customaccount1@setbyenv.com'; const arn = 'arn:aws:iam::123456789012:root'; const { config } = require('../../../lib/Config'); -const { - remoteOverlayIsNewer, - patchConfiguration, -} = require('../../../lib/management/configuration'); +const { remoteOverlayIsNewer, patchConfiguration } = require('../../../lib/management/configuration'); -const { - initManagementDatabase, -} = require('../../../lib/management/index'); +const { initManagementDatabase } = require('../../../lib/management/index'); function initManagementCredentialsMock(cb) { - return metadata.putObjectMD(managementDatabaseName, - tokenConfigurationKey, { privateKey }, {}, - log, error => cb(error)); + return metadata.putObjectMD(managementDatabaseName, tokenConfigurationKey, { privateKey }, {}, log, error => + cb(error), + ); } function getConfig() { @@ -37,14 +31,11 @@ function getConfig() { // Original Config const overlayVersionOriginal = Object.assign({}, config.overlayVersion); const authDataOriginal = Object.assign({}, config.authData).accounts; -const locationConstraintsOriginal = Object.assign({}, - config.locationConstraints); +const locationConstraintsOriginal = Object.assign({}, config.locationConstraints); const restEndpointsOriginal = Object.assign({}, config.restEndpoints); const browserAccessEnabledOriginal = config.browserAccessEnabled; const instanceId = '19683e55-56f7-4a4c-98a7-706c07e4ec30'; -const publicInstanceId = crypto.createHash('sha256') - .update(instanceId) - .digest('hex'); +const publicInstanceId = crypto.createHash('sha256').update(instanceId).digest('hex'); function resetConfig() { config.overlayVersion = overlayVersionOriginal; @@ -63,12 +54,14 @@ function assertConfig(actualConf, expectedConf) { } describe('patchConfiguration', () => { - before(done => initManagementDatabase(log, err => { - if (err) { - return done(err); - } - return initManagementCredentialsMock(done); - })); + before(done => + initManagementDatabase(log, err => { + if (err) { + return done(err); + } + return initManagementCredentialsMock(done); + }), + ); beforeEach(() => { resetConfig(); }); @@ -115,17 +108,21 @@ describe('patchConfiguration', () => { publicInstanceId, browserAccessEnabled: true, authData: { - accounts: [{ - name: userName, - email, - arn, - canonicalID: canonicalId, - shortid, - keys: [{ - access: accessKey, - secret: decryptedSecretKey, - }], - }], + accounts: [ + { + name: userName, + email, + arn, + canonicalID: canonicalId, + shortid, + keys: [ + { + access: accessKey, + secret: decryptedSecretKey, + }, + ], + }, + ], }, locationConstraints: { 'us-east-1': { @@ -141,14 +138,12 @@ describe('patchConfiguration', () => { }, }; assertConfig(actualConf, expectedConf); - assert.deepStrictEqual(actualConf.restEndpoints['1.1.1.1'], - 'us-east-1'); + assert.deepStrictEqual(actualConf.restEndpoints['1.1.1.1'], 'us-east-1'); return done(); }); }); - it('should apply second configuration if version (2) is greater than ' + - 'overlayVersion (1)', done => { + it('should apply second configuration if version (2) is greater than ' + 'overlayVersion (1)', done => { const newConf1 = { version: 1, instanceId, @@ -175,8 +170,7 @@ describe('patchConfiguration', () => { }); }); - it('should not apply the second configuration if version equals ' + - 'overlayVersion', done => { + it('should not apply the second configuration if version equals ' + 'overlayVersion', done => { const newConf1 = { version: 1, instanceId, @@ -205,40 +199,37 @@ describe('patchConfiguration', () => { }); describe('remoteOverlayIsNewer', () => { - it('should return remoteOverlayIsNewer equals false if remote overlay ' + - 'is less than the cached', () => { + it('should return remoteOverlayIsNewer equals false if remote overlay ' + 'is less than the cached', () => { const cachedOverlay = { version: 2, }; const remoteOverlay = { version: 1, }; - const isRemoteOverlayNewer = remoteOverlayIsNewer(cachedOverlay, - remoteOverlay); + const isRemoteOverlayNewer = remoteOverlayIsNewer(cachedOverlay, remoteOverlay); assert.equal(isRemoteOverlayNewer, false); }); - it('should return remoteOverlayIsNewer equals false if remote overlay ' + - 'and the cached one are equal', () => { + it('should return remoteOverlayIsNewer equals false if remote overlay ' + 'and the cached one are equal', () => { const cachedOverlay = { version: 1, }; const remoteOverlay = { version: 1, }; - const isRemoteOverlayNewer = remoteOverlayIsNewer(cachedOverlay, - remoteOverlay); + const isRemoteOverlayNewer = remoteOverlayIsNewer(cachedOverlay, remoteOverlay); assert.equal(isRemoteOverlayNewer, false); }); - it('should return remoteOverlayIsNewer equals true if remote overlay ' + - 'version is greater than the cached one ', () => { - const cachedOverlay = { - version: 0, - }; - const remoteOverlay = { - version: 1, - }; - const isRemoteOverlayNewer = remoteOverlayIsNewer(cachedOverlay, - remoteOverlay); - assert.equal(isRemoteOverlayNewer, true); - }); + it( + 'should return remoteOverlayIsNewer equals true if remote overlay ' + 'version is greater than the cached one ', + () => { + const cachedOverlay = { + version: 0, + }; + const remoteOverlay = { + version: 1, + }; + const isRemoteOverlayNewer = remoteOverlayIsNewer(cachedOverlay, remoteOverlay); + assert.equal(isRemoteOverlayNewer, true); + }, + ); }); diff --git a/tests/unit/management/secureChannel.js b/tests/unit/management/secureChannel.js index dc29bb1b8a..0f63441334 100644 --- a/tests/unit/management/secureChannel.js +++ b/tests/unit/management/secureChannel.js @@ -19,16 +19,16 @@ describe('report handler', () => { }); [ - { value: 'true', result: true }, - { value: 'TRUE', result: true }, - { value: 'tRuE', result: true }, - { value: '1', result: true }, - { value: 'false', result: false }, - { value: 'FALSE', result: false }, - { value: 'FaLsE', result: false }, - { value: '0', result: false }, - { value: 'foo', result: false }, - { value: '', result: true }, + { value: 'true', result: true }, + { value: 'TRUE', result: true }, + { value: 'tRuE', result: true }, + { value: '1', result: true }, + { value: 'false', result: false }, + { value: 'FALSE', result: false }, + { value: 'FaLsE', result: false }, + { value: '0', result: false }, + { value: 'foo', result: false }, + { value: '', result: true }, { value: undefined, result: true }, ].forEach(param => it(`should allow set local file system capability ${param.value}`, () => { @@ -40,6 +40,6 @@ describe('report handler', () => { assert.strictEqual(getCapabilities().locationTypeLocal, param.result); process.env = OLD_ENV; - }) + }), ); }); diff --git a/tests/unit/management/testChannelMessageV0.js b/tests/unit/management/testChannelMessageV0.js index 46a4bb2d2d..f43d6f7687 100644 --- a/tests/unit/management/testChannelMessageV0.js +++ b/tests/unit/management/testChannelMessageV0.js @@ -1,10 +1,6 @@ const assert = require('assert'); -const { - ChannelMessageV0, - MessageType, - TargetType, -} = require('../../../lib/management/ChannelMessageV0'); +const { ChannelMessageV0, MessageType, TargetType } = require('../../../lib/management/ChannelMessageV0'); const { CONFIG_OVERLAY_MESSAGE, diff --git a/tests/unit/multipleBackend/VersioningBackendClient.js b/tests/unit/multipleBackend/VersioningBackendClient.js index 60d8527602..065f34e08f 100644 --- a/tests/unit/multipleBackend/VersioningBackendClient.js +++ b/tests/unit/multipleBackend/VersioningBackendClient.js @@ -7,8 +7,7 @@ const DummyService = require('../DummyService'); const { DummyRequestLogger } = require('../helpers'); const missingVerIdInternalError = errorInstances.InternalError.customizeDescription( - 'Invalid state. Please ensure versioning is enabled ' + - 'in AWS for the location constraint and try again.' + 'Invalid state. Please ensure versioning is enabled ' + 'in AWS for the location constraint and try again.', ); const log = new DummyRequestLogger(); @@ -43,8 +42,7 @@ const s3Config = { }; const assertSuccess = (err, cb) => { - assert.ifError(err, - `Expected success, but got error ${err}`); + assert.ifError(err, `Expected success, but got error ${err}`); cb(); }; @@ -54,26 +52,22 @@ const assertFailure = (err, cb) => { }; const genTests = [ { - msg: 'should return success if supportsVersioning === true ' + - 'and backend versioning is enabled', + msg: 'should return success if supportsVersioning === true ' + 'and backend versioning is enabled', input: { supportsVersioning: true, enableMockVersioning: true }, callback: assertSuccess, }, { - msg: 'should return success if supportsVersioning === false ' + - 'and backend versioning is enabled', + msg: 'should return success if supportsVersioning === false ' + 'and backend versioning is enabled', input: { supportsVersioning: false, enableMockVersioning: true }, callback: assertSuccess, }, { - msg: 'should return error if supportsVersioning === true ' + - 'and backend versioning is disabled', + msg: 'should return error if supportsVersioning === true ' + 'and backend versioning is disabled', input: { supportsVersioning: true, enableMockVersioning: false }, callback: assertFailure, }, { - msg: 'should return success if supportsVersioning === false ' + - 'and backend versioning is disabled', + msg: 'should return success if supportsVersioning === false ' + 'and backend versioning is disabled', input: { supportsVersioning: false, enableMockVersioning: false }, callback: assertSuccess, }, @@ -86,12 +80,13 @@ describe('AwsClient::putObject', () => { testClient = new AwsClient(s3Config); testClient._client = new DummyService({ versioning: true }); }); - genTests.forEach(test => it(test.msg, done => { - testClient._supportsVersioning = test.input.supportsVersioning; - testClient._client.versioning = test.input.enableMockVersioning; - testClient.put('', 0, { bucketName: bucket, objectKey: key }, - reqUID, err => test.callback(err, done)); - })); + genTests.forEach(test => + it(test.msg, done => { + testClient._supportsVersioning = test.input.supportsVersioning; + testClient._client.versioning = test.input.enableMockVersioning; + testClient.put('', 0, { bucketName: bucket, objectKey: key }, reqUID, err => test.callback(err, done)); + }), + ); }); describe('AwsClient::copyObject', () => { @@ -102,13 +97,15 @@ describe('AwsClient::copyObject', () => { testClient._client = new DummyService({ versioning: true }); }); - genTests.forEach(test => it(test.msg, done => { - testClient._supportsVersioning = test.input.supportsVersioning; - testClient._client.versioning = test.input.enableMockVersioning; - testClient.copyObject(copyObjectRequest, null, key, - sourceLocationConstraint, null, config, log, - err => test.callback(err, done)); - })); + genTests.forEach(test => + it(test.msg, done => { + testClient._supportsVersioning = test.input.supportsVersioning; + testClient._client.versioning = test.input.enableMockVersioning; + testClient.copyObject(copyObjectRequest, null, key, sourceLocationConstraint, null, config, log, err => + test.callback(err, done), + ); + }), + ); }); describe('AwsClient::completeMPU', () => { @@ -118,13 +115,14 @@ describe('AwsClient::completeMPU', () => { testClient = new AwsClient(s3Config); testClient._client = new DummyService({ versioning: true }); }); - genTests.forEach(test => it(test.msg, done => { - testClient._supportsVersioning = test.input.supportsVersioning; - testClient._client.versioning = test.input.enableMockVersioning; - const uploadId = 'externalBackendTestUploadId'; - testClient.completeMPU(jsonList, null, key, uploadId, - bucket, log, err => test.callback(err, done)); - })); + genTests.forEach(test => + it(test.msg, done => { + testClient._supportsVersioning = test.input.supportsVersioning; + testClient._client.versioning = test.input.enableMockVersioning; + const uploadId = 'externalBackendTestUploadId'; + testClient.completeMPU(jsonList, null, key, uploadId, bucket, log, err => test.callback(err, done)); + }), + ); }); describe('AwsClient::healthcheck', () => { @@ -159,34 +157,31 @@ describe('AwsClient::healthcheck', () => { const tests = [ { - msg: 'should return success if supportsVersioning === true ' + - 'and backend versioning is enabled', + msg: 'should return success if supportsVersioning === true ' + 'and backend versioning is enabled', input: { supportsVersioning: true, enableMockVersioning: true }, callback: assertSuccessVersioned, }, { - msg: 'should return success if supportsVersioning === false ' + - 'and backend versioning is enabled', + msg: 'should return success if supportsVersioning === false ' + 'and backend versioning is enabled', input: { supportsVersioning: false, enableMockVersioning: true }, callback: assertSuccessNonVersioned, }, { - msg: 'should return error if supportsVersioning === true ' + - ' and backend versioning is disabled', + msg: 'should return error if supportsVersioning === true ' + ' and backend versioning is disabled', input: { supportsVersioning: true, enableMockVersioning: false }, callback: assertFailure, }, { - msg: 'should return success if supportsVersioning === false ' + - 'and backend versioning is disabled', + msg: 'should return success if supportsVersioning === false ' + 'and backend versioning is disabled', input: { supportsVersioning: false, enableMockVersioning: false }, callback: assertSuccessNonVersioned, }, ]; - tests.forEach(test => it(test.msg, done => { - testClient._supportsVersioning = test.input.supportsVersioning; - testClient._client.versioning = test.input.enableMockVersioning; - testClient.healthcheck('backend', - (err, resp) => test.callback(resp.backend, done)); - })); + tests.forEach(test => + it(test.msg, done => { + testClient._supportsVersioning = test.input.supportsVersioning; + testClient._client.versioning = test.input.enableMockVersioning; + testClient.healthcheck('backend', (err, resp) => test.callback(resp.backend, done)); + }), + ); }); diff --git a/tests/unit/multipleBackend/getReplicationBackendDataLocator.js b/tests/unit/multipleBackend/getReplicationBackendDataLocator.js index b27f7595ff..048928725a 100644 --- a/tests/unit/multipleBackend/getReplicationBackendDataLocator.js +++ b/tests/unit/multipleBackend/getReplicationBackendDataLocator.js @@ -1,7 +1,6 @@ const assert = require('assert'); -const getReplicationBackendDataLocator = require( - '../../../lib/api/apiUtils/object/getReplicationBackendDataLocator'); +const getReplicationBackendDataLocator = require('../../../lib/api/apiUtils/object/getReplicationBackendDataLocator'); const locCheckResult = { location: 'spoofbackend', @@ -9,47 +8,45 @@ const locCheckResult = { locationType: 'spoof', }; const repNoMatch = { backends: [{ site: 'nomatch' }] }; -const repMatchPending = { backends: - [{ site: 'spoofbackend', status: 'PENDING', dataVersionId: '' }] }; -const repMatchFailed = { backends: - [{ site: 'spoofbackend', status: 'FAILED', dataVersionId: '' }] }; -const repMatch = { backends: [{ - site: 'spoofbackend', - status: 'COMPLETED', - dataStoreVersionId: 'spoofid' }], +const repMatchPending = { backends: [{ site: 'spoofbackend', status: 'PENDING', dataVersionId: '' }] }; +const repMatchFailed = { backends: [{ site: 'spoofbackend', status: 'FAILED', dataVersionId: '' }] }; +const repMatch = { + backends: [ + { + site: 'spoofbackend', + status: 'COMPLETED', + dataStoreVersionId: 'spoofid', + }, + ], }; -const expDataLocator = [{ - key: locCheckResult.key, - dataStoreName: locCheckResult.location, - dataStoreType: locCheckResult.locationType, - dataStoreVersionId: repMatch.backends[0].dataStoreVersionId, -}]; - +const expDataLocator = [ + { + key: locCheckResult.key, + dataStoreName: locCheckResult.location, + dataStoreType: locCheckResult.locationType, + dataStoreVersionId: repMatch.backends[0].dataStoreVersionId, + }, +]; describe('Replication Backend Compare', () => { it('should return error if no match in replication backends', () => { - const repBackendResult = - getReplicationBackendDataLocator(locCheckResult, repNoMatch); + const repBackendResult = getReplicationBackendDataLocator(locCheckResult, repNoMatch); assert.strictEqual(repBackendResult.error.is.InvalidLocationConstraint, true); }); it('should return a status and reason if backend status is PENDING', () => { - const repBackendResult = - getReplicationBackendDataLocator(locCheckResult, repMatchPending); + const repBackendResult = getReplicationBackendDataLocator(locCheckResult, repMatchPending); assert.strictEqual(repBackendResult.dataLocator, undefined); assert.strictEqual(repBackendResult.status, 'PENDING'); assert.notStrictEqual(repBackendResult.reason, undefined); }); it('should return a status and reason if backend status is FAILED', () => { - const repBackendResult = - getReplicationBackendDataLocator(locCheckResult, repMatchFailed); + const repBackendResult = getReplicationBackendDataLocator(locCheckResult, repMatchFailed); assert.strictEqual(repBackendResult.dataLocator, undefined); assert.strictEqual(repBackendResult.status, 'FAILED'); assert.notStrictEqual(repBackendResult.reason, undefined); }); - it('should return dataLocator obj if backend matches and rep is COMPLETED', - () => { - const repBackendResult = - getReplicationBackendDataLocator(locCheckResult, repMatch); + it('should return dataLocator obj if backend matches and rep is COMPLETED', () => { + const repBackendResult = getReplicationBackendDataLocator(locCheckResult, repMatch); assert.strictEqual(repBackendResult.status, 'COMPLETED'); assert.deepStrictEqual(repBackendResult.dataLocator, expDataLocator); }); diff --git a/tests/unit/multipleBackend/locationConstraintCheck.js b/tests/unit/multipleBackend/locationConstraintCheck.js index 1388e53bbe..dbba7c3fe0 100644 --- a/tests/unit/multipleBackend/locationConstraintCheck.js +++ b/tests/unit/multipleBackend/locationConstraintCheck.js @@ -3,8 +3,7 @@ const assert = require('assert'); const { BucketInfo, BackendInfo } = require('arsenal').models; const DummyRequest = require('../DummyRequest'); const { DummyRequestLogger } = require('../helpers'); -const locationConstraintCheck - = require('../../../lib/api/apiUtils/object/locationConstraintCheck'); +const locationConstraintCheck = require('../../../lib/api/apiUtils/object/locationConstraintCheck'); const memLocation = 'scality-internal-mem'; const fileLocation = 'scality-internal-file'; @@ -18,48 +17,51 @@ const objectKey = 'someobject'; const postBody = Buffer.from('I am a body', 'utf8'); const log = new DummyRequestLogger(); -const testBucket = new BucketInfo(bucketName, owner, ownerDisplayName, - testDate, null, null, null, null, null, null, locationConstraint); +const testBucket = new BucketInfo( + bucketName, + owner, + ownerDisplayName, + testDate, + null, + null, + null, + null, + null, + null, + locationConstraint, +); function createTestRequest(locationConstraint) { - const testRequest = new DummyRequest({ - bucketName, - namespace, - objectKey, - headers: { 'x-amz-meta-scal-location-constraint': locationConstraint }, - url: `/${bucketName}/${objectKey}`, - parsedHost: 'localhost', - }, postBody); + const testRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey, + headers: { 'x-amz-meta-scal-location-constraint': locationConstraint }, + url: `/${bucketName}/${objectKey}`, + parsedHost: 'localhost', + }, + postBody, + ); return testRequest; } describe('Location Constraint Check', () => { - it('should return error if controlling location constraint is ' + - 'not valid', done => { - const backendInfoObj = locationConstraintCheck( - createTestRequest('fail-region'), null, testBucket, log); - assert.strictEqual(backendInfoObj.err.code, 400, - 'Expected "Invalid Argument" code error'); - assert(backendInfoObj.err.is.InvalidArgument, 'Expected "Invalid ' + - 'Argument" error'); + it('should return error if controlling location constraint is ' + 'not valid', done => { + const backendInfoObj = locationConstraintCheck(createTestRequest('fail-region'), null, testBucket, log); + assert.strictEqual(backendInfoObj.err.code, 400, 'Expected "Invalid Argument" code error'); + assert(backendInfoObj.err.is.InvalidArgument, 'Expected "Invalid ' + 'Argument" error'); done(); }); - it('should return instance of BackendInfo with correct ' + - 'locationConstraints', done => { - const backendInfoObj = locationConstraintCheck( - createTestRequest(memLocation), null, testBucket, log); - assert.strictEqual(backendInfoObj.err, null, 'Expected success ' + - `but got error ${backendInfoObj.err}`); + it('should return instance of BackendInfo with correct ' + 'locationConstraints', done => { + const backendInfoObj = locationConstraintCheck(createTestRequest(memLocation), null, testBucket, log); + assert.strictEqual(backendInfoObj.err, null, 'Expected success ' + `but got error ${backendInfoObj.err}`); assert.strictEqual(typeof backendInfoObj.controllingLC, 'string'); - assert.equal(backendInfoObj.backendInfo instanceof BackendInfo, - true); - assert.strictEqual(backendInfoObj. - backendInfo.getObjectLocationConstraint(), memLocation); - assert.strictEqual(backendInfoObj. - backendInfo.getBucketLocationConstraint(), fileLocation); - assert.strictEqual(backendInfoObj.backendInfo.getRequestEndpoint(), - 'localhost'); + assert.equal(backendInfoObj.backendInfo instanceof BackendInfo, true); + assert.strictEqual(backendInfoObj.backendInfo.getObjectLocationConstraint(), memLocation); + assert.strictEqual(backendInfoObj.backendInfo.getBucketLocationConstraint(), fileLocation); + assert.strictEqual(backendInfoObj.backendInfo.getRequestEndpoint(), 'localhost'); done(); }); }); diff --git a/tests/unit/multipleBackend/locationHeaderCheck.js b/tests/unit/multipleBackend/locationHeaderCheck.js index 34d58e0a9b..8a806ee351 100644 --- a/tests/unit/multipleBackend/locationHeaderCheck.js +++ b/tests/unit/multipleBackend/locationHeaderCheck.js @@ -1,8 +1,7 @@ const assert = require('assert'); const { errorInstances } = require('arsenal'); -const locationHeaderCheck = - require('../../../lib/api/apiUtils/object/locationHeaderCheck'); +const locationHeaderCheck = require('../../../lib/api/apiUtils/object/locationHeaderCheck'); const objectKey = 'locationHeaderCheckObject'; const bucketName = 'locationHeaderCheckBucket'; @@ -11,18 +10,22 @@ const testCases = [ { location: 'doesnotexist', expRes: errorInstances.InvalidLocationConstraint.customizeDescription( - 'Invalid location constraint specified in header'), - }, { + 'Invalid location constraint specified in header', + ), + }, + { location: '', expRes: undefined, - }, { + }, + { location: 'awsbackend', expRes: { location: 'awsbackend', key: objectKey, locationType: 'aws_s3', }, - }, { + }, + { location: 'awsbackendmismatch', expRes: { location: 'awsbackendmismatch', @@ -34,11 +37,9 @@ const testCases = [ describe('Location Header Check', () => { testCases.forEach(test => { - it('should return expected result with location constraint header ' + - `set to ${test.location}`, () => { + it('should return expected result with location constraint header ' + `set to ${test.location}`, () => { const headers = { 'x-amz-location-constraint': `${test.location}` }; - const checkRes = - locationHeaderCheck(headers, objectKey, bucketName); + const checkRes = locationHeaderCheck(headers, objectKey, bucketName); assert.deepStrictEqual(checkRes, test.expRes); }); }); diff --git a/tests/unit/policies.js b/tests/unit/policies.js index b1dfdc596e..5e2dda6a23 100644 --- a/tests/unit/policies.js +++ b/tests/unit/policies.js @@ -240,10 +240,7 @@ const apiMatrix = [ headers: { 'x-amz-version-id': '1', }, - expectedPermissions: [ - 's3:PutObject', - 's3:PutObjectVersionTagging', - ], + expectedPermissions: ['s3:PutObject', 's3:PutObjectVersionTagging'], }, { name: 'objectPutACL', @@ -282,7 +279,6 @@ const apiMatrix = [ }, ]; - function prepareDummyRequest(headers = {}) { const request = new DummyRequest({ hostname: 'localhost', @@ -300,13 +296,16 @@ describe('Policies: permission checks for S3 APIs', () => { if (api.name.length === 0) { return; } - const message = `should return ${api.expectedPermissions.join(', ')} in requestContextParams for ${api.name}` + - `${(api.headers && api.headers.length) > 0 ? - ` with headers ${api.headers.map(el => el[0]).join(', ')}` : ''}`; + const message = + `should return ${api.expectedPermissions.join(', ')} in requestContextParams for ${api.name}` + + `${ + (api.headers && api.headers.length) > 0 + ? ` with headers ${api.headers.map(el => el[0]).join(', ')}` + : '' + }`; it(message, () => { const request = prepareDummyRequest(api.headers); - const requestContexts = prepareRequestContexts(api.name, request, - sourceBucket, sourceObject); + const requestContexts = prepareRequestContexts(api.name, request, sourceBucket, sourceObject); const requestedActions = requestContexts.map(rq => rq.getAction()); assert.deepStrictEqual(requestedActions, api.expectedPermissions); }); @@ -320,26 +319,26 @@ describe('Policies: permission checks for S3 APIs', () => { } it('should return s3:PutBucket without any provided header', () => { - assert.deepStrictEqual( - putBucketApiMethods(), - ['bucketPut'], - ); + assert.deepStrictEqual(putBucketApiMethods(), ['bucketPut']); }); - it('should return s3:CreateBucket, s3:PutBucketVersioning and s3:PutBucketObjectLockConfiguration' + - ' with object-lock headers', () => { - assert.deepStrictEqual( - putBucketApiMethods({ 'x-amz-bucket-object-lock-enabled': 'true' }), - ['bucketPut', 'bucketPutObjectLock', 'bucketPutVersioning'], - ); - }); + it( + 'should return s3:CreateBucket, s3:PutBucketVersioning and s3:PutBucketObjectLockConfiguration' + + ' with object-lock headers', + () => { + assert.deepStrictEqual(putBucketApiMethods({ 'x-amz-bucket-object-lock-enabled': 'true' }), [ + 'bucketPut', + 'bucketPutObjectLock', + 'bucketPutVersioning', + ]); + }, + ); - it('should return s3:CreateBucket and s3:PutBucketAcl' + - ' with ACL headers', () => { - assert.deepStrictEqual( - putBucketApiMethods({ 'x-amz-grant-read': 'private' }), - ['bucketPut', 'bucketPutACL'], - ); + it('should return s3:CreateBucket and s3:PutBucketAcl' + ' with ACL headers', () => { + assert.deepStrictEqual(putBucketApiMethods({ 'x-amz-grant-read': 'private' }), [ + 'bucketPut', + 'bucketPutACL', + ]); }); }); }); diff --git a/tests/unit/quotas/scuba/wrapper.js b/tests/unit/quotas/scuba/wrapper.js index 57065ff96a..eddd8a51df 100644 --- a/tests/unit/quotas/scuba/wrapper.js +++ b/tests/unit/quotas/scuba/wrapper.js @@ -44,7 +44,7 @@ describe('ScubaClientImpl', () => { }); it('should disable Scuba if health check returns non-stale data', async () => { - sinon.stub(client, 'healthCheck').resolves({ date: Date.now() - (12 * 60 * 60 * 1000) }); + sinon.stub(client, 'healthCheck').resolves({ date: Date.now() - 12 * 60 * 60 * 1000 }); await client._healthCheck(); @@ -52,7 +52,7 @@ describe('ScubaClientImpl', () => { }); it('should disable Scuba if health check returns stale data', async () => { - sinon.stub(client, 'healthCheck').resolves({ date: Date.now() - (48 * 60 * 60 * 1000) }); + sinon.stub(client, 'healthCheck').resolves({ date: Date.now() - 48 * 60 * 60 * 1000 }); await client._healthCheck(); diff --git a/tests/unit/routes/veeam-routes.js b/tests/unit/routes/veeam-routes.js index f3eb3d9039..02c246755c 100644 --- a/tests/unit/routes/veeam-routes.js +++ b/tests/unit/routes/veeam-routes.js @@ -83,7 +83,8 @@ describe('Veeam routes - comprehensive unit tests', () => { response.end.called = true; response.headersSent = true; // Emit finish event when end is called - const finishHandlers = response.on.getCalls() + const finishHandlers = response.on + .getCalls() .filter(call => call.args[0] === 'finish') .map(call => call.args[1]); finishHandlers.forEach(handler => handler()); @@ -113,12 +114,10 @@ describe('Veeam routes - comprehensive unit tests', () => { assert(logWarnSpy.calledOnce, 'log.warn should have been called once'); const warnCall = logWarnSpy.getCall(0); - assert(warnCall.args[0].includes('UtilizationService returned 404'), - 'warning message should mention 404'); + assert(warnCall.args[0].includes('UtilizationService returned 404'), 'warning message should mention 404'); assert.strictEqual(warnCall.args[1].bucket, 'test-bucket'); - assert(response.writeHead.calledWith(200), - 'should return 200 despite 404 from UtilizationService'); + assert(response.writeHead.calledWith(200), 'should return 200 despite 404 from UtilizationService'); assert(response.end.called, 'response should be ended'); }); @@ -132,8 +131,10 @@ describe('Veeam routes - comprehensive unit tests', () => { await getVeeamFile(request, response, bucketMd, log); - assert(response.headersSent || response.write.called || response.writeHead.called, - 'should send error response for 500 errors'); + assert( + response.headersSent || response.write.called || response.writeHead.called, + 'should send error response for 500 errors', + ); }); it('should handle connection error from UtilizationService and return 500', async () => { @@ -146,8 +147,10 @@ describe('Veeam routes - comprehensive unit tests', () => { await getVeeamFile(request, response, bucketMd, log); - assert(response.headersSent || response.write.called || response.writeHead.called, - 'should send error response for connection errors'); + assert( + response.headersSent || response.write.called || response.writeHead.called, + 'should send error response for connection errors', + ); }); it('should successfully use metrics when UtilizationService returns data', async () => { @@ -168,8 +171,7 @@ describe('Veeam routes - comprehensive unit tests', () => { assert(utilizationStub.calledOnce, 'should call UtilizationService once'); assert(response.end.called, 'response should be ended'); - const lastModifiedCall = response.setHeader.getCalls() - .find(call => call.args[0] === 'Last-Modified'); + const lastModifiedCall = response.setHeader.getCalls().find(call => call.args[0] === 'Last-Modified'); assert(lastModifiedCall, 'Last-Modified header should be set'); assert.strictEqual( @@ -217,8 +219,7 @@ describe('Veeam routes - comprehensive unit tests', () => { await getVeeamFile(request, response, bucketMd, log); assert(logWarnSpy.calledOnce, 'should log warning for 404'); - assert(response.writeHead.calledWith(200), - 'should return 200 with static capacity data for 404'); + assert(response.writeHead.calledWith(200), 'should return 200 with static capacity data for 404'); assert(response.end.called, 'response should be ended'); const warnCall = logWarnSpy.getCall(0); assert(warnCall.args[0].includes('404'), 'warning should mention 404'); @@ -233,8 +234,10 @@ describe('Veeam routes - comprehensive unit tests', () => { await getVeeamFile(request, response, bucketMd, log); - assert(response.headersSent || response.write.called || response.writeHead.called, - 'should send response for metadata errors'); + assert( + response.headersSent || response.write.called || response.writeHead.called, + 'should send response for metadata errors', + ); }); it('should handle tagging query parameter', async () => { @@ -244,8 +247,7 @@ describe('Veeam routes - comprehensive unit tests', () => { await getVeeamFile(request, response, bucketMd, log); - assert(response.writeHead.calledWith(200), - 'should return 200 for tagging query'); + assert(response.writeHead.calledWith(200), 'should return 200 for tagging query'); assert(response.end.called, 'response should be ended'); }); }); @@ -361,8 +363,7 @@ describe('Veeam routes - HEAD request UtilizationService error handling', () => assert(response.setHeader.called, 'should set headers'); assert(response.end.called, 'response should be ended'); - const lastModifiedCall = response.setHeader.getCalls() - .find(call => call.args[0] === 'Last-Modified'); + const lastModifiedCall = response.setHeader.getCalls().find(call => call.args[0] === 'Last-Modified'); assert(lastModifiedCall, 'Last-Modified header should be set'); assert.strictEqual( lastModifiedCall.args[1], @@ -383,8 +384,7 @@ describe('Veeam routes - HEAD request UtilizationService error handling', () => assert(logWarnSpy.calledOnce, 'log.warn should have been called once'); const warnCall = logWarnSpy.getCall(0); - assert(warnCall.args[0].includes('UtilizationService returned 404'), - 'warning message should mention 404'); + assert(warnCall.args[0].includes('UtilizationService returned 404'), 'warning message should mention 404'); assert(response.setHeader.called, 'should set headers'); assert(response.end.called, 'response should be ended'); }); @@ -496,7 +496,8 @@ describe('Veeam routes - LIST request handling', () => { response.once.returns(response); response.end.callsFake(() => { response.end.called = true; - const finishHandlers = response.on.getCalls() + const finishHandlers = response.on + .getCalls() .filter(call => call.args[0] === 'finish') .map(call => call.args[1]); finishHandlers.forEach(handler => handler()); @@ -544,12 +545,11 @@ describe('Veeam routes - LIST request handling', () => { matches.push(match[1]); } - assert.strictEqual(matches.length, 3, - 'should have 3 LastModified entries (system.xml, capacity.xml, folder)'); + assert.strictEqual(matches.length, 3, 'should have 3 LastModified entries (system.xml, capacity.xml, folder)'); const iso8601Regex = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/; - matches.forEach( - value => { assert(iso8601Regex.test(value), `LastModified "${value}" should be in ISO 8601 format`); + matches.forEach(value => { + assert(iso8601Regex.test(value), `LastModified "${value}" should be in ISO 8601 format`); }); }); @@ -565,8 +565,7 @@ describe('Veeam routes - LIST request handling', () => { assert(logWarnSpy.calledOnce, 'log.warn should have been called once'); const warnCall = logWarnSpy.getCall(0); - assert(warnCall.args[0].includes('UtilizationService returned 404'), - 'warning message should mention 404'); + assert(warnCall.args[0].includes('UtilizationService returned 404'), 'warning message should mention 404'); assert(response.writeHead.calledWith(200), 'should return 200 despite 404'); assert(response.end.called, 'response should be ended'); }); @@ -615,31 +614,28 @@ describe('Veeam routes - LIST request handling', () => { assert(response.end.called, 'response should be ended'); }); - it( - 'should list only available files when only SystemInfo is present, without calling UtilizationService', - async () => { - const bucketMdOnlySystem = { - ...bucketMd, - _capabilities: { - VeeamSOSApi: { - SystemInfo: { - ProtocolVersion: '1.0', - ModelName: 'ARTESCA', - LastModified: '2024-01-01T00:00:00.000Z', - }, + it('should list only available files when only SystemInfo is present, without calling UtilizationService', async () => { + const bucketMdOnlySystem = { + ...bucketMd, + _capabilities: { + VeeamSOSApi: { + SystemInfo: { + ProtocolVersion: '1.0', + ModelName: 'ARTESCA', + LastModified: '2024-01-01T00:00:00.000Z', }, }, - }; - metadataStub.callsArgWith(2, null, bucketMdOnlySystem); + }, + }; + metadataStub.callsArgWith(2, null, bucketMdOnlySystem); - const request = createRequest(); - const response = createResponse(); + const request = createRequest(); + const response = createResponse(); - await listVeeamFiles(request, response, bucketMdOnlySystem, log); + await listVeeamFiles(request, response, bucketMdOnlySystem, log); - assert(!utilizationStub.called, 'should not call UtilizationService without CapacityInfo'); - assert(response.writeHead.calledWith(200), 'should return 200'); - assert(response.end.called, 'response should be ended'); - }, - ); + assert(!utilizationStub.called, 'should not call UtilizationService without CapacityInfo'); + assert(response.writeHead.calledWith(200), 'should return 200'); + assert(response.end.called, 'response should be ended'); + }); }); diff --git a/tests/unit/routes/veeam-utils.js b/tests/unit/routes/veeam-utils.js index a76ee65f7c..7f231559a4 100644 --- a/tests/unit/routes/veeam-utils.js +++ b/tests/unit/routes/veeam-utils.js @@ -85,10 +85,7 @@ describe('fetchCapacityMetrics', () => { error500.response = { status: 500 }; utilizationStub.callsArgWith(4, error500); - await assert.rejects( - fetchCapacityMetrics(bucketMd, request, log), - err => err === error500, - ); + await assert.rejects(fetchCapacityMetrics(bucketMd, request, log), err => err === error500); assert(logErrorSpy.calledOnce); assert.strictEqual(logErrorSpy.getCall(0).args[1].bucket, 'test-bucket'); @@ -101,10 +98,7 @@ describe('fetchCapacityMetrics', () => { connError.code = 'ECONNREFUSED'; utilizationStub.callsArgWith(4, connError); - await assert.rejects( - fetchCapacityMetrics(bucketMd, request, log), - err => err === connError, - ); + await assert.rejects(fetchCapacityMetrics(bucketMd, request, log), err => err === connError); assert(logErrorSpy.calledOnce); assert.strictEqual(logErrorSpy.getCall(0).args[1].statusCode, 'ECONNREFUSED'); diff --git a/tests/unit/server.js b/tests/unit/server.js index e0d8604e30..197b457698 100644 --- a/tests/unit/server.js +++ b/tests/unit/server.js @@ -26,7 +26,7 @@ describe('S3Server', () => { internalPort: undefined, internalListenOn: [], metricsListenOn: [], - metricsPort: 8002 + metricsPort: 8002, }; server = new S3Server(config); @@ -38,14 +38,15 @@ describe('S3Server', () => { sinon.restore(); }); - const waitReady = () => new Promise(resolve => { - const interval = setInterval(() => { - if (server.started) { - clearInterval(interval); - resolve(); - } - }, 100); - }); + const waitReady = () => + new Promise(resolve => { + const interval = setInterval(() => { + if (server.started) { + clearInterval(interval); + resolve(); + } + }, 100); + }); describe('initiateStartup', () => { beforeEach(() => { @@ -56,12 +57,13 @@ describe('S3Server', () => { // `sinon` matcher to match when the callback argument actually invokes the expected // function - const wrapperFor = expected => sinon.match(actual => { - const req = uuid.v4(); - const res = uuid.v4(); - actual(req, res); - return expected.calledWith(req, res); - }); + const wrapperFor = expected => + sinon.match(actual => { + const req = uuid.v4(); + const res = uuid.v4(); + actual(req, res); + return expected.calledWith(req, res); + }); it('should start API server with default port if no listenOn is provided', async () => { config.port = 8000; @@ -73,13 +75,12 @@ describe('S3Server', () => { assert.strictEqual(startServerStub.callCount, 2); assert(startServerStub.calledWith(wrapperFor(server.routeRequest), 8000)); assert(startServerStub.calledWith(wrapperFor(server.routeAdminRequest))); - }); - + it('should start API servers from listenOn array', async () => { config.listenOn = [ { port: 8000, ip: '127.0.0.1' }, - { port: 8001, ip: '0.0.0.0' } + { port: 8001, ip: '0.0.0.0' }, ]; config.port = 9999; // Should be ignored since listenOn is provided @@ -93,7 +94,7 @@ describe('S3Server', () => { assert(startServerStub.calledWith(wrapperFor(server.routeAdminRequest))); assert.strictEqual(startServerStub.neverCalledWith(sinon.any, 9999), true); }); - + it('should start internal API server with internalPort if no internalListenOn is provided', async () => { config.internalPort = 9000; @@ -104,11 +105,11 @@ describe('S3Server', () => { assert.strictEqual(startServerStub.callCount, 2); assert(startServerStub.calledWith(wrapperFor(server.internalRouteRequest), 9000)); }); - + it('should start internal API servers from internalListenOn array', async () => { config.internalListenOn = [ { port: 9000, ip: '127.0.0.1' }, - { port: 9001, ip: '0.0.0.0' } + { port: 9001, ip: '0.0.0.0' }, ]; config.internalPort = 9999; // Should be ignored since internalListenOn is provided @@ -122,29 +123,29 @@ describe('S3Server', () => { assert(startServerStub.calledWith(wrapperFor(server.routeAdminRequest))); assert.strictEqual(startServerStub.neverCalledWith(sinon.any, 9999), true); }); - + it('should start metrics server with metricsPort if no metricsListenOn is provided', async () => { config.metricsPort = 8012; server.initiateStartup(log); await waitReady(); - + assert.strictEqual(startServerStub.callCount, 1); assert(startServerStub.calledWith(wrapperFor(server.routeAdminRequest), 8012)); }); - + it('should start metrics servers from metricsListenOn array', async () => { config.metricsListenOn = [ { port: 8002, ip: '127.0.0.1' }, - { port: 8003, ip: '0.0.0.0' } + { port: 8003, ip: '0.0.0.0' }, ]; config.metricsPort = 9999; // Should be ignored since metricsListenOn is provided server.initiateStartup(log); await waitReady(); - + assert.strictEqual(startServerStub.callCount, 2); assert(startServerStub.calledWith(wrapperFor(server.routeAdminRequest), 8002, '127.0.0.1')); assert(startServerStub.calledWith(wrapperFor(server.routeAdminRequest), 8003, '0.0.0.0')); @@ -169,10 +170,10 @@ describe('S3Server', () => { describe('internalRouteRequest', () => { const resp = { - on: () => { }, - setHeader: () => { }, - writeHead: () => { }, - end: () => { }, + on: () => {}, + setHeader: () => {}, + writeHead: () => {}, + end: () => {}, }; let req; @@ -181,7 +182,7 @@ describe('S3Server', () => { req = { headers: {}, socket: { - setNoDelay: () => { }, + setNoDelay: () => {}, }, url: 'http://localhost:8000', }; @@ -219,7 +220,7 @@ describe('S3Server request timeout', () => { beforeEach(() => { sandbox = sinon.createSandbox(); - + // Create a mock server to capture the requestTimeout setting mockServer = { requestTimeout: null, @@ -227,7 +228,7 @@ describe('S3Server request timeout', () => { listen: sandbox.stub(), address: sandbox.stub().returns({ address: '127.0.0.1', port: 8000 }), }; - + // Mock server creation to return our mock sandbox.stub(http, 'createServer').returns(mockServer); sandbox.stub(https, 'createServer').returns(mockServer); @@ -240,12 +241,12 @@ describe('S3Server request timeout', () => { it('should set server.requestTimeout to 0 when starting server', () => { const server = new S3Server({ ...defaultConfig, - https: false + https: false, }); - + // Call _startServer which should set requestTimeout = 0 server._startServer(() => {}, 8000, '127.0.0.1'); - + // Verify that requestTimeout was set to 0 assert.strictEqual(mockServer.requestTimeout, 0); }); diff --git a/tests/unit/testConfigs/allOptsConfig/config.json b/tests/unit/testConfigs/allOptsConfig/config.json index b6519b23a3..1f0bef6c03 100644 --- a/tests/unit/testConfigs/allOptsConfig/config.json +++ b/tests/unit/testConfigs/allOptsConfig/config.json @@ -10,29 +10,34 @@ "127.0.0.2": "us-east-1", "s3.amazonaws.com": "us-east-1" }, - "websiteEndpoints": ["s3-website-us-east-1.amazonaws.com", - "s3-website.us-east-2.amazonaws.com", - "s3-website-us-west-1.amazonaws.com", - "s3-website-us-west-2.amazonaws.com", - "s3-website.ap-south-1.amazonaws.com", - "s3-website.ap-northeast-2.amazonaws.com", - "s3-website-ap-southeast-1.amazonaws.com", - "s3-website-ap-southeast-2.amazonaws.com", - "s3-website-ap-northeast-1.amazonaws.com", - "s3-website.eu-central-1.amazonaws.com", - "s3-website-eu-west-1.amazonaws.com", - "s3-website-sa-east-1.amazonaws.com", - "s3-website.localhost", - "s3-website.scality.test", - "zenkoazuretest.blob.core.windows.net"], - "replicationEndpoints": [{ - "site": "zenko", - "servers": ["127.0.0.1:8000"], - "default": true - }, { - "site": "us-east-2", - "type": "aws_s3" - }], + "websiteEndpoints": [ + "s3-website-us-east-1.amazonaws.com", + "s3-website.us-east-2.amazonaws.com", + "s3-website-us-west-1.amazonaws.com", + "s3-website-us-west-2.amazonaws.com", + "s3-website.ap-south-1.amazonaws.com", + "s3-website.ap-northeast-2.amazonaws.com", + "s3-website-ap-southeast-1.amazonaws.com", + "s3-website-ap-southeast-2.amazonaws.com", + "s3-website-ap-northeast-1.amazonaws.com", + "s3-website.eu-central-1.amazonaws.com", + "s3-website-eu-west-1.amazonaws.com", + "s3-website-sa-east-1.amazonaws.com", + "s3-website.localhost", + "s3-website.scality.test", + "zenkoazuretest.blob.core.windows.net" + ], + "replicationEndpoints": [ + { + "site": "zenko", + "servers": ["127.0.0.1:8000"], + "default": true + }, + { + "site": "us-east-2", + "type": "aws_s3" + } + ], "cdmi": { "host": "localhost", "port": 81, @@ -75,11 +80,11 @@ "recordLogName": "s3-recordlog" }, "mongodb": { - "replicaSetHosts": "localhost:27017,localhost:27018,localhost:27019", - "writeConcern": "majority", - "replicaSet": "rs0", - "readPreference": "primary", - "database": "metadata" + "replicaSetHosts": "localhost:27017,localhost:27018,localhost:27019", + "writeConcern": "majority", + "replicaSet": "rs0", + "readPreference": "primary", + "database": "metadata" }, "certFilePaths": { "key": "tests/unit/testConfigs/allOptsConfig/key.txt", diff --git a/tests/unit/testConfigs/bucketNotifConfigTest.js b/tests/unit/testConfigs/bucketNotifConfigTest.js index 9feded3dac..38118dee44 100644 --- a/tests/unit/testConfigs/bucketNotifConfigTest.js +++ b/tests/unit/testConfigs/bucketNotifConfigTest.js @@ -3,13 +3,15 @@ const { bucketNotifAssert } = require('../../../lib/Config'); describe('bucketNotifAssert', () => { it('should not throw an error if bucket notification config is valid', () => { - bucketNotifAssert([{ - resource: 'target1', - type: 'kafka', - host: 'localhost', - port: 8000, - auth: { user: 'user', password: 'password' }, - }]); + bucketNotifAssert([ + { + resource: 'target1', + type: 'kafka', + host: 'localhost', + port: 8000, + auth: { user: 'user', password: 'password' }, + }, + ]); }); it('should throw an error if bucket notification config is not an array', () => { assert.throws(() => { @@ -20,51 +22,58 @@ describe('bucketNotifAssert', () => { port: 8000, auth: { user: 'user', password: 'password' }, }); - }, - '/bad config: bucket notification configuration must be an array/'); + }, '/bad config: bucket notification configuration must be an array/'); }); it('should throw an error if resource is not a string', () => { assert.throws(() => { - bucketNotifAssert([{ - resource: 12345, - type: 'kafka', - host: 'localhost', - port: 8000, - auth: { user: 'user', password: 'password' }, - }]); + bucketNotifAssert([ + { + resource: 12345, + type: 'kafka', + host: 'localhost', + port: 8000, + auth: { user: 'user', password: 'password' }, + }, + ]); }, '/bad config: bucket notification configuration resource must be a string/'); }); it('should throw an error if type is not a string', () => { assert.throws(() => { - bucketNotifAssert([{ - resource: 'target1', - type: 12345, - host: 'localhost', - port: 8000, - auth: { user: 'user', password: 'password' }, - }]); + bucketNotifAssert([ + { + resource: 'target1', + type: 12345, + host: 'localhost', + port: 8000, + auth: { user: 'user', password: 'password' }, + }, + ]); }, '/bad config: bucket notification configuration type must be a string/'); }); it('should throw an error if host is not a string', () => { assert.throws(() => { - bucketNotifAssert([{ - resource: 'target1', - type: 'kafka', - host: 127.0, - port: 8000, - auth: { user: 'user', password: 'password' }, - }]); + bucketNotifAssert([ + { + resource: 'target1', + type: 'kafka', + host: 127.0, + port: 8000, + auth: { user: 'user', password: 'password' }, + }, + ]); }, '/bad config: bucket notification configuration type must be a string/'); }); it('should throw an error if port is not an integer', () => { assert.throws(() => { - bucketNotifAssert([{ - resource: 'target1', - type: 'kafka', - host: 'localhost', - port: '8000', - auth: { user: 'user', password: 'password' }, - }]); + bucketNotifAssert([ + { + resource: 'target1', + type: 'kafka', + host: 'localhost', + port: '8000', + auth: { user: 'user', password: 'password' }, + }, + ]); }, '/bad config: port must be a positive integer/'); }); // TODO: currently auth is fluid and once a concrete structure is diff --git a/tests/unit/testConfigs/configTest.js b/tests/unit/testConfigs/configTest.js index 881b5f9e1f..afd73f092e 100644 --- a/tests/unit/testConfigs/configTest.js +++ b/tests/unit/testConfigs/configTest.js @@ -6,144 +6,185 @@ const { config } = require('../../../lib/Config'); const userBucketOwner = 'Bart'; const creationDate = new Date().toJSON(); -const serverSideEncryption = { cryptoScheme: 123, algorithm: 'algo', -masterKeyId: 'masterKeyId', mandatory: false }; -const bucketOne = new BucketInfo('bucketone', - userBucketOwner, userBucketOwner, creationDate, - BucketInfo.currentModelVersion()); -const bucketTwo = new BucketInfo('buckettwo', - userBucketOwner, userBucketOwner, creationDate, - BucketInfo.currentModelVersion()); -const bucketOnetWithEncryption = new BucketInfo('bucketone', - userBucketOwner, userBucketOwner, creationDate, - BucketInfo.currentModelVersion(), undefined, undefined, undefined, - serverSideEncryption); -const bucketTwoWithEncryption = new BucketInfo('buckettwo', - userBucketOwner, userBucketOwner, creationDate, - BucketInfo.currentModelVersion(), undefined, undefined, undefined, - serverSideEncryption); +const serverSideEncryption = { cryptoScheme: 123, algorithm: 'algo', masterKeyId: 'masterKeyId', mandatory: false }; +const bucketOne = new BucketInfo( + 'bucketone', + userBucketOwner, + userBucketOwner, + creationDate, + BucketInfo.currentModelVersion(), +); +const bucketTwo = new BucketInfo( + 'buckettwo', + userBucketOwner, + userBucketOwner, + creationDate, + BucketInfo.currentModelVersion(), +); +const bucketOnetWithEncryption = new BucketInfo( + 'bucketone', + userBucketOwner, + userBucketOwner, + creationDate, + BucketInfo.currentModelVersion(), + undefined, + undefined, + undefined, + serverSideEncryption, +); +const bucketTwoWithEncryption = new BucketInfo( + 'buckettwo', + userBucketOwner, + userBucketOwner, + creationDate, + BucketInfo.currentModelVersion(), + undefined, + undefined, + undefined, + serverSideEncryption, +); const results = [ - { sourceLocationConstraintName: 'azurebackend', - destLocationConstraintName: 'azurebackend', - sourceBucketMD: bucketOne, - destBucketMD: bucketOne, - boolExpected: true, - description: 'same bucket metadata', - }, - { sourceLocationConstraintName: 'azurebackend2', - destLocationConstraintName: 'azurebackend2', - sourceBucketMD: bucketOne, - destBucketMD: bucketOne, - boolExpected: true, - description: 'same bucket metadata', - }, - { sourceLocationConstraintName: 'awsbackend', - destLocationConstraintName: 'awsbackend', - sourceBucketMD: bucketOne, - destBucketMD: bucketOne, - boolExpected: true, - description: 'same bucket metadata', - }, - { sourceLocationConstraintName: 'awsbackend', - destLocationConstraintName: 'awsbackend2', - sourceBucketMD: bucketOne, - destBucketMD: bucketOne, - boolExpected: true, - description: 'same bucket metadata', - }, - { sourceLocationConstraintName: 'awsbackend2', - destLocationConstraintName: 'awsbackend2', - sourceBucketMD: bucketOne, - destBucketMD: bucketOne, - boolExpected: true, - description: 'same bucket metadata', - }, - { sourceLocationConstraintName: 'scality-internal-mem', - destLocationConstraintName: 'scality-internal-mem', - sourceBucketMD: bucketOne, - destBucketMD: bucketOne, - boolExpected: false, - description: 'same bucket metadata', - }, - { sourceLocationConstraintName: 'scality-internal-mem', - destLocationConstraintName: 'azurebackend', - sourceBucketMD: bucketOne, - destBucketMD: bucketOne, - boolExpected: false, - description: 'same bucket metadata', - }, - { sourceLocationConstraintName: 'azurebackend', - destLocationConstraintName: 'scality-internal-mem', - sourceBucketMD: bucketOne, - destBucketMD: bucketOne, - boolExpected: false, - description: 'same bucket metadata', - }, - { sourceLocationConstraintName: 'awsbackend', - destLocationConstraintName: 'scality-internal-mem', - sourceBucketMD: bucketOne, - destBucketMD: bucketOne, - boolExpected: false, - description: 'same bucket metadata', - }, - { sourceLocationConstraintName: 'scality-internal-mem', - destLocationConstraintName: 'awsbackend', - sourceBucketMD: bucketOne, - destBucketMD: bucketOne, - boolExpected: false, - description: 'same bucket metadata', - }, - { sourceLocationConstraintName: 'azurebackend', - destLocationConstraintName: 'awsbackend', - sourceBucketMD: bucketOne, - destBucketMD: bucketOne, - boolExpected: false, - description: 'same bucket metadata', - }, - { sourceLocationConstraintName: 'azurebackend', - destLocationConstraintName: 'azurebackend2', - sourceBucketMD: bucketOne, - destBucketMD: bucketOne, - boolExpected: false, - description: 'same bucket metadata', - }, - { sourceLocationConstraintName: 'azurebackend', - destLocationConstraintName: 'azurebackend', - sourceBucketMD: bucketOne, - destBucketMD: bucketTwo, - boolExpected: true, - description: 'different non-encrypted bucket metadata', - }, - { sourceLocationConstraintName: 'azurebackend', - destLocationConstraintName: 'azurebackend', - sourceBucketMD: bucketOnetWithEncryption, - destBucketMD: bucketOnetWithEncryption, - boolExpected: true, - description: 'same encrypted bucket metadata', - }, - { sourceLocationConstraintName: 'azurebackend', - destLocationConstraintName: 'azurebackend', - sourceBucketMD: bucketOnetWithEncryption, - destBucketMD: bucketTwoWithEncryption, - boolExpected: false, - description: 'different encrypted bucket metadata', - }, + { + sourceLocationConstraintName: 'azurebackend', + destLocationConstraintName: 'azurebackend', + sourceBucketMD: bucketOne, + destBucketMD: bucketOne, + boolExpected: true, + description: 'same bucket metadata', + }, + { + sourceLocationConstraintName: 'azurebackend2', + destLocationConstraintName: 'azurebackend2', + sourceBucketMD: bucketOne, + destBucketMD: bucketOne, + boolExpected: true, + description: 'same bucket metadata', + }, + { + sourceLocationConstraintName: 'awsbackend', + destLocationConstraintName: 'awsbackend', + sourceBucketMD: bucketOne, + destBucketMD: bucketOne, + boolExpected: true, + description: 'same bucket metadata', + }, + { + sourceLocationConstraintName: 'awsbackend', + destLocationConstraintName: 'awsbackend2', + sourceBucketMD: bucketOne, + destBucketMD: bucketOne, + boolExpected: true, + description: 'same bucket metadata', + }, + { + sourceLocationConstraintName: 'awsbackend2', + destLocationConstraintName: 'awsbackend2', + sourceBucketMD: bucketOne, + destBucketMD: bucketOne, + boolExpected: true, + description: 'same bucket metadata', + }, + { + sourceLocationConstraintName: 'scality-internal-mem', + destLocationConstraintName: 'scality-internal-mem', + sourceBucketMD: bucketOne, + destBucketMD: bucketOne, + boolExpected: false, + description: 'same bucket metadata', + }, + { + sourceLocationConstraintName: 'scality-internal-mem', + destLocationConstraintName: 'azurebackend', + sourceBucketMD: bucketOne, + destBucketMD: bucketOne, + boolExpected: false, + description: 'same bucket metadata', + }, + { + sourceLocationConstraintName: 'azurebackend', + destLocationConstraintName: 'scality-internal-mem', + sourceBucketMD: bucketOne, + destBucketMD: bucketOne, + boolExpected: false, + description: 'same bucket metadata', + }, + { + sourceLocationConstraintName: 'awsbackend', + destLocationConstraintName: 'scality-internal-mem', + sourceBucketMD: bucketOne, + destBucketMD: bucketOne, + boolExpected: false, + description: 'same bucket metadata', + }, + { + sourceLocationConstraintName: 'scality-internal-mem', + destLocationConstraintName: 'awsbackend', + sourceBucketMD: bucketOne, + destBucketMD: bucketOne, + boolExpected: false, + description: 'same bucket metadata', + }, + { + sourceLocationConstraintName: 'azurebackend', + destLocationConstraintName: 'awsbackend', + sourceBucketMD: bucketOne, + destBucketMD: bucketOne, + boolExpected: false, + description: 'same bucket metadata', + }, + { + sourceLocationConstraintName: 'azurebackend', + destLocationConstraintName: 'azurebackend2', + sourceBucketMD: bucketOne, + destBucketMD: bucketOne, + boolExpected: false, + description: 'same bucket metadata', + }, + { + sourceLocationConstraintName: 'azurebackend', + destLocationConstraintName: 'azurebackend', + sourceBucketMD: bucketOne, + destBucketMD: bucketTwo, + boolExpected: true, + description: 'different non-encrypted bucket metadata', + }, + { + sourceLocationConstraintName: 'azurebackend', + destLocationConstraintName: 'azurebackend', + sourceBucketMD: bucketOnetWithEncryption, + destBucketMD: bucketOnetWithEncryption, + boolExpected: true, + description: 'same encrypted bucket metadata', + }, + { + sourceLocationConstraintName: 'azurebackend', + destLocationConstraintName: 'azurebackend', + sourceBucketMD: bucketOnetWithEncryption, + destBucketMD: bucketTwoWithEncryption, + boolExpected: false, + description: 'different encrypted bucket metadata', + }, ]; describe('Testing Config.js function: ', () => { results.forEach(result => { - it(`should return ${result.boolExpected} if source location ` + - `constraint === ${result.sourceLocationConstraintName} ` + - 'and destination location constraint ===' + - ` ${result.destLocationConstraintName} and ${result.description}`, - done => { - const isCopy = utils.externalBackendCopy(config, - result.sourceLocationConstraintName, - result.destLocationConstraintName, result.sourceBucketMD, - result.destBucketMD); - assert.strictEqual(isCopy, result.boolExpected); - done(); - }); + it( + `should return ${result.boolExpected} if source location ` + + `constraint === ${result.sourceLocationConstraintName} ` + + 'and destination location constraint ===' + + ` ${result.destLocationConstraintName} and ${result.description}`, + done => { + const isCopy = utils.externalBackendCopy( + config, + result.sourceLocationConstraintName, + result.destLocationConstraintName, + result.sourceBucketMD, + result.destBucketMD, + ); + assert.strictEqual(isCopy, result.boolExpected); + done(); + }, + ); }); }); diff --git a/tests/unit/testConfigs/locConstraintAssert.js b/tests/unit/testConfigs/locConstraintAssert.js index f56fbec6e5..e9051d4b31 100644 --- a/tests/unit/testConfigs/locConstraintAssert.js +++ b/tests/unit/testConfigs/locConstraintAssert.js @@ -7,23 +7,30 @@ class LocationConstraint { this.objectId = objectId; this.legacyAwsBehavior = legacyAwsBehavior || false; this.sizeLimitGB = sizeLimit || undefined; - this.details = Object.assign({}, { - awsEndpoint: 's3.amazonaws.com', - bucketName: 'tester', - credentialsProfile: 'default', - region: 'us-west-1', - }, details || {}); + this.details = Object.assign( + {}, + { + awsEndpoint: 's3.amazonaws.com', + bucketName: 'tester', + credentialsProfile: 'default', + region: 'us-west-1', + }, + details || {}, + ); } } function getAzureDetails(replaceParams) { - return Object.assign({ - azureStorageEndpoint: 'https://fakeaccountname.blob.core.fake.net/', - azureStorageAccountName: 'fakeaccountname', - azureStorageAccessKey: 'Fake00Key123', - bucketMatch: false, - azureContainerName: 'test', - }, replaceParams); + return Object.assign( + { + azureStorageEndpoint: 'https://fakeaccountname.blob.core.fake.net/', + azureStorageAccountName: 'fakeaccountname', + azureStorageAccessKey: 'Fake00Key123', + bucketMatch: false, + azureContainerName: 'test', + }, + replaceParams, + ); } // FIXME: most of tests using a line-wrapped regexp are broken, @@ -35,331 +42,301 @@ describe('locationConstraintAssert', () => { it('should throw error if locationConstraints is not an object', () => { assert.throws(() => { locationConstraintAssert(''); - }, - /bad config: locationConstraints must be an object/); + }, /bad config: locationConstraints must be an object/); }); it('should throw error if any location constraint is not an object', () => { - assert.throws(() => { - locationConstraintAssert({ notObject: '' }); - }, - err => { - assert.strictEqual(err.message, 'bad config: ' + - 'locationConstraints[region] must be an object'); - return true; - }); + assert.throws( + () => { + locationConstraintAssert({ notObject: '' }); + }, + err => { + assert.strictEqual(err.message, 'bad config: ' + 'locationConstraints[region] must be an object'); + return true; + }, + ); }); it('should throw error if type is not a string', () => { const locationConstraint = new LocationConstraint(42, 'locId'); - assert.throws(() => { - locationConstraintAssert({ 'scality-east': locationConstraint }); - }, - /bad config: locationConstraints[region].type is mandatory/ + - /and must be a string/); + assert.throws( + () => { + locationConstraintAssert({ 'scality-east': locationConstraint }); + }, + /bad config: locationConstraints[region].type is mandatory/ + /and must be a string/, + ); }); it('should throw error if type is not mem/file/scality/dmf/crr', () => { - const locationConstraint = new LocationConstraint( - 'notSupportedType', 'locId'); - assert.throws(() => { - locationConstraintAssert({ 'scality-east': locationConstraint }); - }, - /bad config: locationConstraints[region].type must be/ + - /one of mem,file,scality,tlp,crr/); + const locationConstraint = new LocationConstraint('notSupportedType', 'locId'); + assert.throws( + () => { + locationConstraintAssert({ 'scality-east': locationConstraint }); + }, + /bad config: locationConstraints[region].type must be/ + /one of mem,file,scality,tlp,crr/, + ); }); it('should throw error if legacyAwsBehavior is not a boolean', () => { - const locationConstraint = new LocationConstraint( - 'scality', 'locId', 42); - assert.throws(() => { - locationConstraintAssert({ 'scality-east': locationConstraint }); - }, - /bad config: locationConstraints[region].legacyAwsBehavior / + - /is mandatory and must be a boolean/); + const locationConstraint = new LocationConstraint('scality', 'locId', 42); + assert.throws( + () => { + locationConstraintAssert({ 'scality-east': locationConstraint }); + }, + /bad config: locationConstraints[region].legacyAwsBehavior / + /is mandatory and must be a boolean/, + ); }); it('should throw error if details is not an object', () => { - const locationConstraint = - new LocationConstraint('scality', 'locId', false, 42); - assert.throws(() => { - locationConstraintAssert({ 'scality-east': locationConstraint }); - }, - /bad config: locationConstraints[region].details is / + - /mandatory and must be an object/); + const locationConstraint = new LocationConstraint('scality', 'locId', false, 42); + assert.throws( + () => { + locationConstraintAssert({ 'scality-east': locationConstraint }); + }, + /bad config: locationConstraints[region].details is / + /mandatory and must be an object/, + ); }); it('should throw error if awsEndpoint is not a string', () => { - const locationConstraint = new LocationConstraint( - 'scality', 'locId', false, - { - awsEndpoint: 42, - }); + const locationConstraint = new LocationConstraint('scality', 'locId', false, { + awsEndpoint: 42, + }); assert.throws(() => { locationConstraintAssert({ 'scality-east': locationConstraint }); - }, - /bad config: awsEndpoint must be a string/); + }, /bad config: awsEndpoint must be a string/); }); it('should throw error if bucketName is not a string', () => { - const locationConstraint = new LocationConstraint( - 'scality', 'locId', false, - { - awsEndpoint: 's3.amazonaws.com', - bucketName: 42, - }); + const locationConstraint = new LocationConstraint('scality', 'locId', false, { + awsEndpoint: 's3.amazonaws.com', + bucketName: 42, + }); assert.throws(() => { locationConstraintAssert({ 'scality-east': locationConstraint }); - }, - /bad config: bucketName must be a string/); + }, /bad config: bucketName must be a string/); }); it('should throw error if credentialsProfile is not a string', () => { - const locationConstraint = new LocationConstraint( - 'scality', 'locId', false, - { - awsEndpoint: 's3.amazonaws.com', - bucketName: 'premadebucket', - credentialsProfile: 42, - }); + const locationConstraint = new LocationConstraint('scality', 'locId', false, { + awsEndpoint: 's3.amazonaws.com', + bucketName: 'premadebucket', + credentialsProfile: 42, + }); assert.throws(() => { locationConstraintAssert({ 'scality-east': locationConstraint }); - }, - /bad config: credentialsProfile must be a string/); + }, /bad config: credentialsProfile must be a string/); }); it('should throw error if region is not a string', () => { - const locationConstraint = new LocationConstraint( - 'scality', 'locId', false, - { - awsEndpoint: 's3.amazonaws.com', - bucketName: 'premadebucket', - credentialsProfile: 'zenko', - region: 42, - }); + const locationConstraint = new LocationConstraint('scality', 'locId', false, { + awsEndpoint: 's3.amazonaws.com', + bucketName: 'premadebucket', + credentialsProfile: 'zenko', + region: 42, + }); assert.throws(() => { locationConstraintAssert({ 'scality-east': locationConstraint }); - }, - /bad config: region must be a string/); + }, /bad config: region must be a string/); }); it('should throw error if us-east-1 not specified', () => { const locationConstraint = new LocationConstraint(); assert.throws(() => { locationConstraintAssert({ 'not-us-east-1': locationConstraint }); - }, - '/bad locationConfig: must ' + - 'include us-east-1 as a locationConstraint/'); + }, '/bad locationConfig: must ' + 'include us-east-1 as a locationConstraint/'); }); it('should not throw error for a valid azure location constraint', () => { const usEast1 = new LocationConstraint(undefined, 'locId1'); - const locationConstraint = new LocationConstraint( - 'azure', 'locId2', true, - getAzureDetails()); + const locationConstraint = new LocationConstraint('azure', 'locId2', true, getAzureDetails()); assert.doesNotThrow(() => { - locationConstraintAssert({ 'azurefaketest': locationConstraint, - 'us-east-1': usEast1 }); - }, - '/should not throw for a valid azure location constraint/'); + locationConstraintAssert({ azurefaketest: locationConstraint, 'us-east-1': usEast1 }); + }, '/should not throw for a valid azure location constraint/'); }); - it('should throw error if type is azure and azureContainerName is ' + - 'not specified', () => { + it('should throw error if type is azure and azureContainerName is ' + 'not specified', () => { const usEast1 = new LocationConstraint(undefined, 'locId1'); const locationConstraint = new LocationConstraint( - 'azure', 'locId2', true, - getAzureDetails({ azureContainerName: undefined })); + 'azure', + 'locId2', + true, + getAzureDetails({ azureContainerName: undefined }), + ); assert.throws(() => { locationConstraintAssert({ 'us-east-1': usEast1, - 'azurefaketest': locationConstraint, + azurefaketest: locationConstraint, }); - }, - '/bad location constraint: ' + - '"azurefaketest" azureContainerName must be defined/'); + }, '/bad location constraint: ' + '"azurefaketest" azureContainerName must be defined/'); }); - it('should throw error if type is azure and azureContainerName is ' + - 'invalid value', () => { + it('should throw error if type is azure and azureContainerName is ' + 'invalid value', () => { const usEast1 = new LocationConstraint(undefined, 'locId1'); const locationConstraint = new LocationConstraint( - 'azure', 'locId2', true, - getAzureDetails({ azureContainerName: '.' })); + 'azure', + 'locId2', + true, + getAzureDetails({ azureContainerName: '.' }), + ); assert.throws(() => { locationConstraintAssert({ 'us-east-1': usEast1, - 'azurefaketest': locationConstraint, + azurefaketest: locationConstraint, }); - }, - '/bad location constraint: "azurefaketest" ' + - 'azureContainerName is an invalid container name/'); + }, '/bad location constraint: "azurefaketest" ' + 'azureContainerName is an invalid container name/'); }); - it('should throw error if type is azure and azureStorageEndpoint ' + - 'is not specified', () => { + it('should throw error if type is azure and azureStorageEndpoint ' + 'is not specified', () => { const usEast1 = new LocationConstraint(undefined, 'locId1'); const locationConstraint = new LocationConstraint( - 'azure', 'locId2', true, - getAzureDetails({ azureStorageEndpoint: undefined })); - assert.throws(() => { - locationConstraintAssert({ - 'us-east-1': usEast1, - 'azurefaketest': locationConstraint, - }); - }, - '/bad location constraint: "azurefaketest" ' + - 'azureStorageEndpoint must be set in locationConfig ' + - 'or environment variable/'); + 'azure', + 'locId2', + true, + getAzureDetails({ azureStorageEndpoint: undefined }), + ); + assert.throws( + () => { + locationConstraintAssert({ + 'us-east-1': usEast1, + azurefaketest: locationConstraint, + }); + }, + '/bad location constraint: "azurefaketest" ' + + 'azureStorageEndpoint must be set in locationConfig ' + + 'or environment variable/', + ); }); - it('should throw error if type is azure and azureStorageAccountName ' + - 'is not specified', () => { + it('should throw error if type is azure and azureStorageAccountName ' + 'is not specified', () => { const usEast1 = new LocationConstraint(undefined, 'locId1'); const locationConstraint = new LocationConstraint( - 'azure', 'locId2', true, - getAzureDetails({ azureStorageAccountName: undefined })); - assert.throws(() => { - locationConstraintAssert({ - 'us-east-1': usEast1, - 'azurefaketest': locationConstraint, - }); - }, - '/bad location constraint: "azurefaketest" ' + - 'azureStorageAccountName must be set in locationConfig ' + - 'or environment variable/'); + 'azure', + 'locId2', + true, + getAzureDetails({ azureStorageAccountName: undefined }), + ); + assert.throws( + () => { + locationConstraintAssert({ + 'us-east-1': usEast1, + azurefaketest: locationConstraint, + }); + }, + '/bad location constraint: "azurefaketest" ' + + 'azureStorageAccountName must be set in locationConfig ' + + 'or environment variable/', + ); }); - it('should throw error if type is azure and azureStorageAccountName ' + - 'is invalid value', () => { + it('should throw error if type is azure and azureStorageAccountName ' + 'is invalid value', () => { const usEast1 = new LocationConstraint(undefined, 'locId1'); const locationConstraint = new LocationConstraint( - 'azure', 'locId2', true, - getAzureDetails({ azureStorageAccountName: 'invalid!!!' })); + 'azure', + 'locId2', + true, + getAzureDetails({ azureStorageAccountName: 'invalid!!!' }), + ); assert.throws(() => { locationConstraintAssert({ 'us-east-1': usEast1, - 'azurefaketest': locationConstraint, + azurefaketest: locationConstraint, }); - }, - '/bad location constraint: "azurefaketest" ' + - 'azureStorageAccountName "invalid!!!" is an invalid value/'); + }, '/bad location constraint: "azurefaketest" ' + 'azureStorageAccountName "invalid!!!" is an invalid value/'); }); - it('should throw error if type is azure and azureStorageAccessKey ' + - 'is not specified', () => { + it('should throw error if type is azure and azureStorageAccessKey ' + 'is not specified', () => { const usEast1 = new LocationConstraint(undefined, 'locId1'); const locationConstraint = new LocationConstraint( - 'azure', 'locId2', true, - getAzureDetails({ azureStorageAccessKey: undefined })); - assert.throws(() => { - locationConstraintAssert({ - 'us-east-1': usEast1, - 'azurefaketest': locationConstraint, - }); - }, - '/bad location constraint: "azurefaketest" ' + - 'azureStorageAccessKey must be set in locationConfig ' + - 'or environment variable/'); + 'azure', + 'locId2', + true, + getAzureDetails({ azureStorageAccessKey: undefined }), + ); + assert.throws( + () => { + locationConstraintAssert({ + 'us-east-1': usEast1, + azurefaketest: locationConstraint, + }); + }, + '/bad location constraint: "azurefaketest" ' + + 'azureStorageAccessKey must be set in locationConfig ' + + 'or environment variable/', + ); }); - it('should throw error if type is azure and azureStorageAccessKey ' + - 'is not a valid base64 string', () => { + it('should throw error if type is azure and azureStorageAccessKey ' + 'is not a valid base64 string', () => { const usEast1 = new LocationConstraint(undefined, 'locId1'); const locationConstraint = new LocationConstraint( - 'azure', 'locId2', true, - getAzureDetails({ azureStorageAccessKey: 'invalid!!!' })); + 'azure', + 'locId2', + true, + getAzureDetails({ azureStorageAccessKey: 'invalid!!!' }), + ); assert.throws(() => { locationConstraintAssert({ 'us-east-1': usEast1, - 'azurefaketest': locationConstraint, + azurefaketest: locationConstraint, }); - }, - '/bad location constraint: "azurefaketest" ' + - 'azureStorageAccessKey is not a valid base64 string/'); + }, '/bad location constraint: "azurefaketest" ' + 'azureStorageAccessKey is not a valid base64 string/'); }); it('should set https to true by default', () => { const usEast1 = new LocationConstraint(undefined, 'locId1'); - const locationConstraint = new LocationConstraint( - 'aws_s3', 'locId2', true); + const locationConstraint = new LocationConstraint('aws_s3', 'locId2', true); assert.doesNotThrow(() => { locationConstraintAssert({ 'us-east-1': usEast1, - 'awshttpsDefault': locationConstraint, + awshttpsDefault: locationConstraint, }); - }, '/bad location constraint awshttpsDefault,' + - 'incorrect default config for https'); - assert.strictEqual(locationConstraint.details.https, true, - 'https config should be true'); + }, '/bad location constraint awshttpsDefault,' + 'incorrect default config for https'); + assert.strictEqual(locationConstraint.details.https, true, 'https config should be true'); }); it('should override default if https is set to false', () => { const usEast1 = new LocationConstraint(undefined, 'locId1'); - const locationConstraint = new LocationConstraint( - 'aws_s3', 'locId2', true, { - https: false, - }); + const locationConstraint = new LocationConstraint('aws_s3', 'locId2', true, { + https: false, + }); assert.doesNotThrow(() => { locationConstraintAssert({ 'us-east-1': usEast1, - 'awshttpsFalse': locationConstraint, + awshttpsFalse: locationConstraint, }); - }, '/bad location constraint awshttpsFalse,' + - 'incorrect config for https'); - assert.strictEqual(locationConstraint.details.https, false, - 'https config should be false'); + }, '/bad location constraint awshttpsFalse,' + 'incorrect config for https'); + assert.strictEqual(locationConstraint.details.https, false, 'https config should be false'); }); it('should set pathStyle config option to false by default', () => { const usEast1 = new LocationConstraint(undefined, 'locId1'); - const locationConstraint = new LocationConstraint( - 'aws_s3', 'locId2', true); + const locationConstraint = new LocationConstraint('aws_s3', 'locId2', true); assert.doesNotThrow(() => { locationConstraintAssert({ 'us-east-1': usEast1, - 'awsdefaultstyle': locationConstraint, + awsdefaultstyle: locationConstraint, }); }, '/bad location constraint, unable to set default config'); - assert.strictEqual(locationConstraint.details.pathStyle, false, - 'pathstyle config should be false'); + assert.strictEqual(locationConstraint.details.pathStyle, false, 'pathstyle config should be false'); }); it('should override default if pathStyle is set to true', () => { const usEast1 = new LocationConstraint(undefined, 'locId1'); - const locationConstraint = new LocationConstraint( - 'aws_s3', 'locId2', true, - { pathStyle: true }); + const locationConstraint = new LocationConstraint('aws_s3', 'locId2', true, { pathStyle: true }); assert.doesNotThrow(() => { locationConstraintAssert({ 'us-east-1': usEast1, - 'awspathstyle': locationConstraint, + awspathstyle: locationConstraint, }); }, '/bad location constraint, unable to set pathSytle config'); - assert.strictEqual(locationConstraint.details.pathStyle, true, - 'pathstyle config should be true'); + assert.strictEqual(locationConstraint.details.pathStyle, true, 'pathstyle config should be true'); }); it('should throw error if sizeLimitGB is not a number', () => { const usEast1 = new LocationConstraint(undefined, 'locId1'); - const locationConstraint = new LocationConstraint( - 'aws_s3', 'locId2', true, - null, true); + const locationConstraint = new LocationConstraint('aws_s3', 'locId2', true, null, true); assert.throws(() => { locationConstraintAssert({ 'us-east-1': usEast1, - 'awsstoragesizelimit': locationConstraint, + awsstoragesizelimit: locationConstraint, }); - }, - '/bad config: locationConstraints[region].sizeLimitGB ' + - 'must be a number (in gigabytes)'); + }, '/bad config: locationConstraints[region].sizeLimitGB ' + 'must be a number (in gigabytes)'); }); it('should throw error if objectId is not set', () => { const usEast1 = new LocationConstraint(undefined, 'locId1'); - const locationConstraint = new LocationConstraint( - 'azure', undefined, true, - getAzureDetails()); + const locationConstraint = new LocationConstraint('azure', undefined, true, getAzureDetails()); assert.throws(() => { - locationConstraintAssert({ 'azurefaketest': locationConstraint, - 'us-east-1': usEast1 }); - }, - '/bad config: locationConstraints[region].objectId is mandatory ' + - 'and must be a unique string across locations'); + locationConstraintAssert({ azurefaketest: locationConstraint, 'us-east-1': usEast1 }); + }, '/bad config: locationConstraints[region].objectId is mandatory ' + 'and must be a unique string across locations'); }); it('should throw error if objectId is duplicated', () => { const usEast1 = new LocationConstraint(undefined, 'locId1'); - const locationConstraint = new LocationConstraint( - 'azure', 'locId1', true, - getAzureDetails()); + const locationConstraint = new LocationConstraint('azure', 'locId1', true, getAzureDetails()); assert.throws(() => { - locationConstraintAssert({ 'azurefaketest': locationConstraint, - 'us-east-1': usEast1 }); - }, - '/bad config: location constraint objectId "locId1" is not unique ' + - 'across configured locations'); + locationConstraintAssert({ azurefaketest: locationConstraint, 'us-east-1': usEast1 }); + }, '/bad config: location constraint objectId "locId1" is not unique ' + 'across configured locations'); }); }); diff --git a/tests/unit/testConfigs/parseKmsAWS.js b/tests/unit/testConfigs/parseKmsAWS.js index c075375192..c466bbee77 100644 --- a/tests/unit/testConfigs/parseKmsAWS.js +++ b/tests/unit/testConfigs/parseKmsAWS.js @@ -338,6 +338,6 @@ describe('parseKmsAWS TLS section', () => { assert(readFileSyncStub.calledWith(path.join(basePath, caPath))); }); - assert(readFileSyncStub.callCount === (keyPaths.length + certPaths.length + caPaths.length)); + assert(readFileSyncStub.callCount === keyPaths.length + certPaths.length + caPaths.length); }); }); diff --git a/tests/unit/testConfigs/parseRedisConfig.spec.js b/tests/unit/testConfigs/parseRedisConfig.spec.js index f585bcec7a..2583cc266c 100644 --- a/tests/unit/testConfigs/parseRedisConfig.spec.js +++ b/tests/unit/testConfigs/parseRedisConfig.spec.js @@ -31,8 +31,7 @@ describe('parseRedisConfig', () => { input: { host: 'localhost', port: 6479, - retry: { - }, + retry: {}, }, }, { @@ -195,8 +194,7 @@ describe('parseRedisConfig', () => { host: 'localhost', port: 6479, retry: { - connectBackoff: { - }, + connectBackoff: {}, }, }, }, diff --git a/tests/unit/testConfigs/parseSproxydConfig.js b/tests/unit/testConfigs/parseSproxydConfig.js index ae22ba94c6..4f057a7771 100644 --- a/tests/unit/testConfigs/parseSproxydConfig.js +++ b/tests/unit/testConfigs/parseSproxydConfig.js @@ -19,16 +19,14 @@ function makeSproxydConf(bootstrap, chordCos, sproxydPath) { describe('parseSproxydConfig', () => { it('should return a parsed config if valid', () => { - const sproxydConf = parseSproxydConfig(makeSproxydConf( - ['localhost:8181'], null, '/arc')); + const sproxydConf = parseSproxydConfig(makeSproxydConf(['localhost:8181'], null, '/arc')); assert.deepStrictEqual(sproxydConf, { bootstrap: ['localhost:8181'], path: '/arc', }); }); it('should return a parsed config with chordCos if valid', () => { - const sproxydConf = parseSproxydConfig(makeSproxydConf( - ['localhost:8181'], '3', '/arc')); + const sproxydConf = parseSproxydConfig(makeSproxydConf(['localhost:8181'], '3', '/arc')); assert.deepStrictEqual(sproxydConf, { bootstrap: ['localhost:8181'], path: '/arc', @@ -40,8 +38,7 @@ describe('parseSproxydConfig', () => { parseSproxydConfig(makeSproxydConf('localhost:8181')); }); }); - it('should throw an error if bootstrap array does not contain strings', - () => { + it('should throw an error if bootstrap array does not contain strings', () => { assert.throws(() => { parseSproxydConfig(makeSproxydConf([8181])); }); diff --git a/tests/unit/testConfigs/requestsConfigTest.js b/tests/unit/testConfigs/requestsConfigTest.js index 41dbdee516..9569534a33 100644 --- a/tests/unit/testConfigs/requestsConfigTest.js +++ b/tests/unit/testConfigs/requestsConfigTest.js @@ -5,11 +5,9 @@ describe('requestsConfigAssert', () => { it('should not throw an error if there is no requests config', () => { assert.doesNotThrow(() => { requestsConfigAssert({}); - }, - 'should not throw an error if there is no requests config'); + }, 'should not throw an error if there is no requests config'); }); - it('should not throw an error if requests config via proxy is set to false', - () => { + it('should not throw an error if requests config via proxy is set to false', () => { assert.doesNotThrow(() => { requestsConfigAssert({ viaProxy: false, @@ -17,25 +15,24 @@ describe('requestsConfigAssert', () => { extractClientIPFromHeader: '', extractProtocolFromHeader: '', }); - }, - 'shouldnt throw an error if requests config via proxy is set to false'); + }, 'shouldnt throw an error if requests config via proxy is set to false'); }); - it('should not throw an error if requests config via proxy is true, ' + - 'trustedProxyCIDRs & extractClientIPFromHeader & ' + - 'extractProtocolFromHeader are set', () => { - assert.doesNotThrow(() => { - requestsConfigAssert({ - viaProxy: true, - trustedProxyCIDRs: ['123.123.123.123'], - extractClientIPFromHeader: 'x-forwarded-for', - extractProtocolFromHeader: 'x-forwarded-proto', - }); + it( + 'should not throw an error if requests config via proxy is true, ' + + 'trustedProxyCIDRs & extractClientIPFromHeader & ' + + 'extractProtocolFromHeader are set', + () => { + assert.doesNotThrow(() => { + requestsConfigAssert({ + viaProxy: true, + trustedProxyCIDRs: ['123.123.123.123'], + extractClientIPFromHeader: 'x-forwarded-for', + extractProtocolFromHeader: 'x-forwarded-proto', + }); + }, 'should not throw an error if requests config ' + 'via proxy is set correctly'); }, - 'should not throw an error if requests config ' + - 'via proxy is set correctly'); - }); - it('should throw an error if requests.viaProxy is not a boolean', - () => { + ); + it('should throw an error if requests.viaProxy is not a boolean', () => { assert.throws(() => { requestsConfigAssert({ viaProxy: 1, @@ -43,93 +40,97 @@ describe('requestsConfigAssert', () => { extractClientIPFromHeader: 'x-forwarded-for', extractProtocolFromHeader: 'x-forwarded-proto', }); - }, - '/config: invalid requests configuration. viaProxy must be a ' + - 'boolean/'); + }, '/config: invalid requests configuration. viaProxy must be a ' + 'boolean/'); }); - it('should throw an error if requests.trustedProxyCIDRs is not an array', - () => { - assert.throws(() => { - requestsConfigAssert({ - viaProxy: true, - trustedProxyCIDRs: 1, - extractClientIPFromHeader: 'x-forwarded-for', - extractProtocolFromHeader: 'x-forwarded-proto', - }); - }, - '/config: invalid requests configuration. ' + - 'trustedProxyCIDRs must be set if viaProxy is set to true ' + - 'and must be an array/'); + it('should throw an error if requests.trustedProxyCIDRs is not an array', () => { + assert.throws( + () => { + requestsConfigAssert({ + viaProxy: true, + trustedProxyCIDRs: 1, + extractClientIPFromHeader: 'x-forwarded-for', + extractProtocolFromHeader: 'x-forwarded-proto', + }); + }, + '/config: invalid requests configuration. ' + + 'trustedProxyCIDRs must be set if viaProxy is set to true ' + + 'and must be an array/', + ); }); - it('should throw an error if requests.trustedProxyCIDRs array is empty', - () => { - assert.throws(() => { - requestsConfigAssert({ - viaProxy: true, - trustedProxyCIDRs: [], - extractClientIPFromHeader: 'x-forwarded-for', - extractProtocolFromHeader: 'x-forwarded-proto', - }); - }, - '/config: invalid requests configuration. ' + - 'trustedProxyCIDRs must be set if viaProxy is set to true ' + - 'and must be an array/'); + it('should throw an error if requests.trustedProxyCIDRs array is empty', () => { + assert.throws( + () => { + requestsConfigAssert({ + viaProxy: true, + trustedProxyCIDRs: [], + extractClientIPFromHeader: 'x-forwarded-for', + extractProtocolFromHeader: 'x-forwarded-proto', + }); + }, + '/config: invalid requests configuration. ' + + 'trustedProxyCIDRs must be set if viaProxy is set to true ' + + 'and must be an array/', + ); }); - it('should throw an error if requests.extractClientIPFromHeader ' + - 'is not a string', () => { - assert.throws(() => { - requestsConfigAssert({ - viaProxy: true, - trustedProxyCIDRs: [], - extractClientIPFromHeader: 1, - extractProtocolFromHeader: 'x-forwarded-proto', - }); - }, - '/config: invalid requests configuration. ' + - 'extractClientIPFromHeader must be set if viaProxy is ' + - 'set to true and must be a string/'); + it('should throw an error if requests.extractClientIPFromHeader ' + 'is not a string', () => { + assert.throws( + () => { + requestsConfigAssert({ + viaProxy: true, + trustedProxyCIDRs: [], + extractClientIPFromHeader: 1, + extractProtocolFromHeader: 'x-forwarded-proto', + }); + }, + '/config: invalid requests configuration. ' + + 'extractClientIPFromHeader must be set if viaProxy is ' + + 'set to true and must be a string/', + ); }); - it('should throw an error if requests.extractProtocolFromHeader ' + - 'is not a string', () => { - assert.throws(() => { - requestsConfigAssert({ - viaProxy: true, - trustedProxyCIDRs: [], - extractClientIPFromHeader: 'x-forwarded-for', - extractProtocolFromHeader: 1, - }); - }, - '/config: invalid requests configuration. ' + - 'extractProtocolFromHeader must be set if viaProxy is ' + - 'set to true and must be a string/'); + it('should throw an error if requests.extractProtocolFromHeader ' + 'is not a string', () => { + assert.throws( + () => { + requestsConfigAssert({ + viaProxy: true, + trustedProxyCIDRs: [], + extractClientIPFromHeader: 'x-forwarded-for', + extractProtocolFromHeader: 1, + }); + }, + '/config: invalid requests configuration. ' + + 'extractProtocolFromHeader must be set if viaProxy is ' + + 'set to true and must be a string/', + ); }); - it('should throw an error if requests.extractClientIPFromHeader ' + - 'is empty', () => { - assert.throws(() => { - requestsConfigAssert({ - viaProxy: true, - trustedProxyCIDRs: [], - extractClientIPFromHeader: '', - extractProtocolFromHeader: 'x-forwarded-proto', - }); - }, - '/config: invalid requests configuration. ' + - 'extractClientIPFromHeader must be set if viaProxy is ' + - 'set to true and must be a string/'); + it('should throw an error if requests.extractClientIPFromHeader ' + 'is empty', () => { + assert.throws( + () => { + requestsConfigAssert({ + viaProxy: true, + trustedProxyCIDRs: [], + extractClientIPFromHeader: '', + extractProtocolFromHeader: 'x-forwarded-proto', + }); + }, + '/config: invalid requests configuration. ' + + 'extractClientIPFromHeader must be set if viaProxy is ' + + 'set to true and must be a string/', + ); }); - it('should throw an error if requests.extractProtocolFromHeader ' + - 'is empty', () => { - assert.throws(() => { - requestsConfigAssert({ - viaProxy: true, - trustedProxyCIDRs: [], - extractClientIPFromHeader: 'x-forwarded-for', - extractProtocolFromHeader: '', - }); - }, - '/config: invalid requests configuration. ' + - 'extractProtocolFromHeader must be set if viaProxy is ' + - 'set to true and must be a string/'); + it('should throw an error if requests.extractProtocolFromHeader ' + 'is empty', () => { + assert.throws( + () => { + requestsConfigAssert({ + viaProxy: true, + trustedProxyCIDRs: [], + extractClientIPFromHeader: 'x-forwarded-for', + extractProtocolFromHeader: '', + }); + }, + '/config: invalid requests configuration. ' + + 'extractProtocolFromHeader must be set if viaProxy is ' + + 'set to true and must be a string/', + ); }); it('should lowercase the extractClientIPFromHeader and extractProtocolFromHeader values', () => { const config = { diff --git a/tests/unit/utils/aclUtils.js b/tests/unit/utils/aclUtils.js index 9985ca8af1..c0b397b2a4 100644 --- a/tests/unit/utils/aclUtils.js +++ b/tests/unit/utils/aclUtils.js @@ -1,39 +1,33 @@ const assert = require('assert'); const aclUtils = require('../../../lib/utilities/aclUtils'); - describe('checkGrantHeaderValidity for acls', () => { const tests = [ { it: 'should allow valid x-amz-grant-read grant', headers: { - 'x-amz-grant-read': - 'uri=http://acs.amazonaws.com/groups/global/AllUsers', + 'x-amz-grant-read': 'uri=http://acs.amazonaws.com/groups/global/AllUsers', }, result: true, }, { it: 'should allow valid x-amz-grant-write grant', headers: { - 'x-amz-grant-write': - 'emailaddress=user2@example.com', + 'x-amz-grant-write': 'emailaddress=user2@example.com', }, result: true, }, { it: 'should allow valid x-amz-grant-read-acp grant', headers: { - 'x-amz-grant-read-acp': - 'emailaddress=superuser@example.com', + 'x-amz-grant-read-acp': 'emailaddress=superuser@example.com', }, result: true, }, { it: 'should allow valid x-amz-grant-write-acp grant', headers: { - 'x-amz-grant-write-acp': - 'id=79a59df900b949e55d96a1e6' + - '98fbacedfd6e09d98eacf8f8d5218e7cd47ef2be', + 'x-amz-grant-write-acp': 'id=79a59df900b949e55d96a1e6' + '98fbacedfd6e09d98eacf8f8d5218e7cd47ef2be', }, result: true, }, @@ -41,50 +35,44 @@ describe('checkGrantHeaderValidity for acls', () => { it: 'should allow valid x-amz-grant-full-control grant', headers: { 'x-amz-grant-full-control': - 'id=79a59df900b949e55d96a1e6' + - '98fbacedfd6e09d98eacf8f8d5218e7cd47ef2be,' + - 'emailaddress=foo@bar.com', + 'id=79a59df900b949e55d96a1e6' + + '98fbacedfd6e09d98eacf8f8d5218e7cd47ef2be,' + + 'emailaddress=foo@bar.com', }, result: true, }, { it: 'should deny grant without equal sign', headers: { - 'x-amz-grant-full-control': - 'id79a59df900b949e55d96a1e6' + - '98fbacedfd6e09d98eacf8f8d5218e7cd47ef2be', + 'x-amz-grant-full-control': 'id79a59df900b949e55d96a1e6' + '98fbacedfd6e09d98eacf8f8d5218e7cd47ef2be', }, result: false, }, { it: 'should deny grant with bad uri', headers: { - 'x-amz-grant-full-control': - 'uri=http://totallymadeup', + 'x-amz-grant-full-control': 'uri=http://totallymadeup', }, result: false, }, { it: 'should deny grant with bad emailaddress', headers: { - 'x-amz-grant-read': - 'emailaddress=invalidemail.com', + 'x-amz-grant-read': 'emailaddress=invalidemail.com', }, result: false, }, { it: 'should deny grant with bad canonicalID', headers: { - 'x-amz-grant-write': - 'id=123', + 'x-amz-grant-write': 'id=123', }, result: false, }, { it: 'should deny grant with bad type of identifier', headers: { - 'x-amz-grant-write': - 'madeupidentifier=123', + 'x-amz-grant-write': 'madeupidentifier=123', }, result: false, }, @@ -92,8 +80,7 @@ describe('checkGrantHeaderValidity for acls', () => { tests.forEach(test => { it(test.it, () => { - const actualResult = - aclUtils.checkGrantHeaderValidity(test.headers); + const actualResult = aclUtils.checkGrantHeaderValidity(test.headers); assert.strictEqual(actualResult, test.result); }); }); diff --git a/tests/unit/utils/bucketEncryption.js b/tests/unit/utils/bucketEncryption.js index 63cb33df9a..230f18f891 100644 --- a/tests/unit/utils/bucketEncryption.js +++ b/tests/unit/utils/bucketEncryption.js @@ -1,14 +1,12 @@ const metadata = require('../../../lib/metadata/wrapper'); - function templateSSEConfig({ algorithm, keyId }) { const xml = []; xml.push(` - ` - ); + `); if (algorithm) { xml.push(`${algorithm}`); diff --git a/tests/unit/utils/checkReadLocation.js b/tests/unit/utils/checkReadLocation.js index 6756f9bff4..34ce3fd712 100644 --- a/tests/unit/utils/checkReadLocation.js +++ b/tests/unit/utils/checkReadLocation.js @@ -1,8 +1,7 @@ const assert = require('assert'); const { ConfigObject } = require('../../../lib/Config'); -const checkReadLocation = - require('../../../lib/api/apiUtils/object/checkReadLocation'); +const checkReadLocation = require('../../../lib/api/apiUtils/object/checkReadLocation'); const locationConstraints = { bucketmatch: { @@ -38,14 +37,12 @@ describe('Testing checkReadLocation', () => { }); it('should return null if location does not exist', () => { - const testResult = checkReadLocation( - config, 'nonexistloc', key, bucket); + const testResult = checkReadLocation(config, 'nonexistloc', key, bucket); assert.deepStrictEqual(testResult, null); }); it('should return correct results for bucketMatch true location', () => { - const testResult = checkReadLocation( - config, 'bucketmatch', key, bucket); + const testResult = checkReadLocation(config, 'bucketmatch', key, bucket); const expectedResult = { location: 'bucketmatch', key, @@ -55,8 +52,7 @@ describe('Testing checkReadLocation', () => { }); it('should return correct results for bucketMatch false location', () => { - const testResult = checkReadLocation( - config, 'nobucketmatch', key, bucket); + const testResult = checkReadLocation(config, 'nobucketmatch', key, bucket); const expectedResult = { location: 'nobucketmatch', key: `${bucket}/${key}`, diff --git a/tests/unit/utils/collectResponseHeaders.js b/tests/unit/utils/collectResponseHeaders.js index 75cd3c134f..ced3c18e48 100644 --- a/tests/unit/utils/collectResponseHeaders.js +++ b/tests/unit/utils/collectResponseHeaders.js @@ -1,6 +1,5 @@ const assert = require('assert'); -const collectResponseHeaders = - require('../../../lib/utilities/collectResponseHeaders'); +const collectResponseHeaders = require('../../../lib/utilities/collectResponseHeaders'); describe('Middleware: Collect Response Headers', () => { it('should be able to set replication status when config is set', () => { @@ -21,22 +20,19 @@ describe('Middleware: Collect Response Headers', () => { }; const headers = collectResponseHeaders(objectMD); assert.deepStrictEqual(headers['x-amz-replication-status'], 'COMPLETED'); - assert.deepStrictEqual(headers['x-amz-meta-us-east-1-replication-status'], - 'COMPLETED'); + assert.deepStrictEqual(headers['x-amz-meta-us-east-1-replication-status'], 'COMPLETED'); assert.deepStrictEqual(headers['x-amz-meta-us-east-1-version-id'], '123'); - assert.deepStrictEqual(headers['x-amz-meta-us-west-2-replication-status'], - 'COMPLETED'); + assert.deepStrictEqual(headers['x-amz-meta-us-west-2-replication-status'], 'COMPLETED'); assert.deepStrictEqual(headers['x-amz-meta-us-west-2-version-id'], undefined); }); - + [ { md: { replicationInfo: null }, test: 'when config is not set' }, { md: {}, test: 'for older objects' }, ].forEach(item => { it(`should skip replication header ${item.test}`, () => { const headers = collectResponseHeaders(item.md); - assert.deepStrictEqual(headers['x-amz-replication-status'], - undefined); + assert.deepStrictEqual(headers['x-amz-replication-status'], undefined); }); }); @@ -45,19 +41,16 @@ describe('Middleware: Collect Response Headers', () => { assert.strictEqual(headers['Accept-Ranges'], 'bytes'); }); - it('should return an undefined value when x-amz-website-redirect-location' + - ' is empty', () => { + it('should return an undefined value when x-amz-website-redirect-location' + ' is empty', () => { const objectMD = { 'x-amz-website-redirect-location': '' }; const headers = collectResponseHeaders(objectMD); - assert.strictEqual(headers['x-amz-website-redirect-location'], - undefined); + assert.strictEqual(headers['x-amz-website-redirect-location'], undefined); }); it('should return the (nonempty) value of WebsiteRedirectLocation', () => { const obj = { 'x-amz-website-redirect-location': 'google.com' }; const headers = collectResponseHeaders(obj); - assert.strictEqual(headers['x-amz-website-redirect-location'], - 'google.com'); + assert.strictEqual(headers['x-amz-website-redirect-location'], 'google.com'); }); it('should not set flag when transition not in progress', () => { diff --git a/tests/unit/utils/gcpMpuHelpers.js b/tests/unit/utils/gcpMpuHelpers.js index 99496a6a3d..6820dda281 100644 --- a/tests/unit/utils/gcpMpuHelpers.js +++ b/tests/unit/utils/gcpMpuHelpers.js @@ -39,8 +39,7 @@ describe('GcpUtils MPU Helper Functions:', () => { tests.forEach(test => { it(test.it, () => { const { partNumber, phase } = test.input; - assert.strictEqual(createMpuKey( - key, uploadId, partNumber, phase), test.output); + assert.strictEqual(createMpuKey(key, uploadId, partNumber, phase), test.output); }); }); }); @@ -56,9 +55,7 @@ describe('GcpUtils MPU Helper Functions:', () => { tests.forEach(test => { it(test.it, () => { const { phase, size } = test.input; - assert.deepStrictEqual(createMpuList( - { Key: key, UploadId: uploadId }, phase, size), - test.output); + assert.deepStrictEqual(createMpuList({ Key: key, UploadId: uploadId }, phase, size), test.output); }); }); }); diff --git a/tests/unit/utils/gcpTaggingHelpers.js b/tests/unit/utils/gcpTaggingHelpers.js index d2169156d0..b7018bba42 100644 --- a/tests/unit/utils/gcpTaggingHelpers.js +++ b/tests/unit/utils/gcpTaggingHelpers.js @@ -1,10 +1,8 @@ const assert = require('assert'); const { errorInstances, storage } = require('arsenal'); const { gcpTaggingPrefix } = require('../../../constants'); -const { genPutTagObj } = - require('../../../tests/functional/raw-node/utils/gcpUtils'); -const { processTagSet, stripTags, retrieveTags, getPutTagsMetadata } = - storage.data.external.GcpUtils; +const { genPutTagObj } = require('../../../tests/functional/raw-node/utils/gcpUtils'); +const { processTagSet, stripTags, retrieveTags, getPutTagsMetadata } = storage.data.external.GcpUtils; const maxTagSize = 10; const validTagSet = genPutTagObj(2); @@ -36,26 +34,24 @@ describe('GcpUtils Tagging Helper Functions:', () => { { it: 'should return error for invalid tag set size', input: invalidSizeTagSet, - output: errorInstances.BadRequest.customizeDescription( - 'Object tags cannot be greater than 10'), + output: errorInstances.BadRequest.customizeDescription('Object tags cannot be greater than 10'), }, { it: 'should return error for duplicate tag keys', input: invalidDuplicateTagSet, output: errorInstances.InvalidTag.customizeDescription( - 'Cannot provide multiple Tags with the same key'), + 'Cannot provide multiple Tags with the same key', + ), }, { it: 'should return error for invalid "key" value', input: invalidKeyTagSet, - output: errorInstances.InvalidTag.customizeDescription( - 'The TagKey provided is too long, 129'), + output: errorInstances.InvalidTag.customizeDescription('The TagKey provided is too long, 129'), }, { it: 'should return error for invalid "value" value', input: invalidValueTagSet, - output: errorInstances.InvalidTag.customizeDescription( - 'The TagValue provided is too long, 257'), + output: errorInstances.InvalidTag.customizeDescription('The TagValue provided is too long, 257'), }, { it: 'should return empty tag object when input is undefined', @@ -123,8 +119,7 @@ describe('GcpUtils Tagging Helper Functions:', () => { describe('getPutTagsMetadata', () => { const tests = [ { - it: 'should return correct object when' + - ' given a tag query string and a metadata obj', + it: 'should return correct object when' + ' given a tag query string and a metadata obj', input: { metadata: Object.assign({}, onlyMetadata), tagQuery }, output: tagMetadata, }, @@ -139,7 +134,8 @@ describe('GcpUtils Tagging Helper Functions:', () => { output: onlyMetadata, }, { - it: 'should return metadata with correct tag properties ' + + it: + 'should return metadata with correct tag properties ' + 'if given a metdata with prior tags and query string', input: { metadata: Object.assign({}, withPriorTags), tagQuery }, output: tagMetadata, @@ -148,8 +144,7 @@ describe('GcpUtils Tagging Helper Functions:', () => { tests.forEach(test => { it(test.it, () => { const { metadata, tagQuery } = test.input; - assert.deepStrictEqual( - getPutTagsMetadata(metadata, tagQuery), test.output); + assert.deepStrictEqual(getPutTagsMetadata(metadata, tagQuery), test.output); }); }); }); diff --git a/tests/unit/utils/lifecycleHelpers.js b/tests/unit/utils/lifecycleHelpers.js index b9399b00ac..a35cff8e4d 100644 --- a/tests/unit/utils/lifecycleHelpers.js +++ b/tests/unit/utils/lifecycleHelpers.js @@ -33,7 +33,8 @@ function getLifecycleXml() { const days2 = 1; const action3 = 'AbortIncompleteMultipartUpload'; const days3 = 30; - return '' + '' + `${id1}` + @@ -61,7 +62,8 @@ function getLifecycleXml() { `${tags[0].value}` + `<${action1}>${days1}` + '' + - ''; + '' + ); } module.exports = { diff --git a/tests/unit/utils/monitoring.js b/tests/unit/utils/monitoring.js index 96ef99070b..3a2ed260c5 100644 --- a/tests/unit/utils/monitoring.js +++ b/tests/unit/utils/monitoring.js @@ -7,8 +7,12 @@ const monitoring = require('../../../lib/utilities/monitoringHandler'); describe('Monitoring: endpoint', () => { const sandbox = sinon.createSandbox(); const res = { - writeHead(/* result, headers */) { return this; }, - write(/* body */) { return this; }, + writeHead(/* result, headers */) { + return this; + }, + write(/* body */) { + return this; + }, end(/* body */) {}, }; monitoring.collectDefaultMetrics(); @@ -23,9 +27,20 @@ describe('Monitoring: endpoint', () => { }); async function fetchMetrics(req, res) { - await new Promise(resolve => monitoring.monitoringHandler(null, req, { - ...res, end: (...body) => { res.end(...body); resolve(); } - }, null)); + await new Promise(resolve => + monitoring.monitoringHandler( + null, + req, + { + ...res, + end: (...body) => { + res.end(...body); + resolve(); + }, + }, + null, + ), + ); } it('should return an error is method is not GET', async () => { @@ -80,20 +95,28 @@ describe('Monitoring: endpoint', () => { }); function parseMetric(metrics, name, labels) { - const labelsString = Object.entries(labels).map(e => `${e[0]}="${e[1]}"`).join(','); + const labelsString = Object.entries(labels) + .map(e => `${e[0]}="${e[1]}"`) + .join(','); const metric = metrics.match(new RegExp(`^${name}{${labelsString}} (.*)$`, 'm')); return metric ? metric[1] : null; } function parseHttpRequestSize(metrics, action = 'putObject') { - const value = parseMetric(metrics, 's3_cloudserver_http_request_size_bytes_sum', - { method: 'PUT', action, code: '200' }); + const value = parseMetric(metrics, 's3_cloudserver_http_request_size_bytes_sum', { + method: 'PUT', + action, + code: '200', + }); return value ? parseInt(value, 10) : 0; } function parseHttpResponseSize(metrics, action = 'getObject') { - const value = parseMetric(metrics, 's3_cloudserver_http_response_size_bytes_sum', - { method: 'GET', action, code: '200' }); + const value = parseMetric(metrics, 's3_cloudserver_http_response_size_bytes_sum', { + method: 'GET', + action, + code: '200', + }); return value ? parseInt(value, 10) : 0; } @@ -101,8 +124,7 @@ describe('Monitoring: endpoint', () => { await fetchMetrics({ method: 'GET', url: '/metrics' }, res); const requestSize = parseHttpRequestSize(res.end.args[0][0]); - monitoring.promMetrics('PUT', 'stuff', '200', - 'putObject', 2357, 3572, false, null, 5723); + monitoring.promMetrics('PUT', 'stuff', '200', 'putObject', 2357, 3572, false, null, 5723); await fetchMetrics({ method: 'GET', url: '/metrics' }, res); assert(parseHttpRequestSize(res.end.args[1][0]) === requestSize + 2357); @@ -112,8 +134,7 @@ describe('Monitoring: endpoint', () => { await fetchMetrics({ method: 'GET', url: '/metrics' }, res); const responseSize = parseHttpResponseSize(res.end.args[0][0]); - monitoring.promMetrics('GET', 'stuff', '200', - 'getObject', 7532); + monitoring.promMetrics('GET', 'stuff', '200', 'getObject', 7532); await fetchMetrics({ method: 'GET', url: '/metrics' }, res); assert(parseHttpResponseSize(res.end.args[1][0]) === responseSize + 7532); diff --git a/tests/unit/utils/mpuUtils.js b/tests/unit/utils/mpuUtils.js index 397b07c5b9..824d9af6b3 100644 --- a/tests/unit/utils/mpuUtils.js +++ b/tests/unit/utils/mpuUtils.js @@ -4,14 +4,11 @@ const crypto = require('crypto'); const xml2js = require('xml2js'); const DummyRequest = require('../DummyRequest'); -const initiateMultipartUpload - = require('../../../lib/api/initiateMultipartUpload'); +const initiateMultipartUpload = require('../../../lib/api/initiateMultipartUpload'); const objectPutPart = require('../../../lib/api/objectPutPart'); -const completeMultipartUpload - = require('../../../lib/api/completeMultipartUpload'); +const completeMultipartUpload = require('../../../lib/api/completeMultipartUpload'); -const { makeAuthInfo } - = require('../helpers'); +const { makeAuthInfo } = require('../helpers'); const canonicalID = 'accessKey1'; const authInfo = makeAuthInfo(canonicalID); @@ -35,31 +32,35 @@ function createinitiateMPURequest(namespace, bucketName, objectKey) { } function createPutPartRequest(namespace, bucketName, objectKey, partNumber, testUploadId) { - const request = new DummyRequest({ - bucketName, - namespace, - objectKey, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - url: `/${objectKey}?partNumber=${partNumber}&uploadId=${testUploadId}`, - query: { - partNumber, - uploadId: testUploadId, + const request = new DummyRequest( + { + bucketName, + namespace, + objectKey, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: `/${objectKey}?partNumber=${partNumber}&uploadId=${testUploadId}`, + query: { + partNumber, + uploadId: testUploadId, + }, + calculatedHash, + actionImplicitDenies: false, }, - calculatedHash, - actionImplicitDenies: false, - }, partBody); + partBody, + ); return request; } function createCompleteRequest(namespace, bucketName, objectKey, testUploadId) { - // only suports a single part for now - const completeBody = '' + - '' + - '1' + - `"${calculatedHash}"` + - '' + - ''; + // only suports a single part for now + const completeBody = + '' + + '' + + '1' + + `"${calculatedHash}"` + + '' + + ''; const request = { bucketName, @@ -78,34 +79,32 @@ function createCompleteRequest(namespace, bucketName, objectKey, testUploadId) { function createMPU(namespace, bucketName, objectKey, logger, cb) { let testUploadId; - async.waterfall([ - next => { - const initiateMPURequest = createinitiateMPURequest(namespace, - bucketName, - objectKey); - initiateMultipartUpload(authInfo, initiateMPURequest, logger, next); - }, - (result, corsHeaders, next) => xml2js.parseString(result, next), - (json, next) => { - testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; - const partRequest = - createPutPartRequest(namespace, bucketName, objectKey, 1, testUploadId); - objectPutPart(authInfo, partRequest, undefined, logger, next); + async.waterfall( + [ + next => { + const initiateMPURequest = createinitiateMPURequest(namespace, bucketName, objectKey); + initiateMultipartUpload(authInfo, initiateMPURequest, logger, next); + }, + (result, corsHeaders, next) => xml2js.parseString(result, next), + (json, next) => { + testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; + const partRequest = createPutPartRequest(namespace, bucketName, objectKey, 1, testUploadId); + objectPutPart(authInfo, partRequest, undefined, logger, next); + }, + (hexDigest, corsHeaders, next) => { + const completeRequest = createCompleteRequest(namespace, bucketName, objectKey, testUploadId); + completeMultipartUpload(authInfo, completeRequest, logger, next); + }, + ], + err => { + assert.ifError(err); + cb(null, testUploadId); }, - (hexDigest, corsHeaders, next) => { - const completeRequest = - createCompleteRequest(namespace, bucketName, objectKey, testUploadId); - completeMultipartUpload(authInfo, completeRequest, logger, next); - }, - ], err => { - assert.ifError(err); - cb(null, testUploadId); - }); + ); return testUploadId; } - module.exports = { createPutPartRequest, createCompleteRequest, diff --git a/tests/unit/utils/multipleBackendGateway.js b/tests/unit/utils/multipleBackendGateway.js index c62b89bff6..868af60ad5 100644 --- a/tests/unit/utils/multipleBackendGateway.js +++ b/tests/unit/utils/multipleBackendGateway.js @@ -1,9 +1,7 @@ const assert = require('assert'); const { checkExternalBackend } = require('arsenal').storage.data.external.backendUtils; const sinon = require('sinon'); -const awsLocations = [ - 'awsbackend', -]; +const awsLocations = ['awsbackend']; const statusSuccess = { versioningStatus: 'Enabled', @@ -32,37 +30,46 @@ describe('Testing _checkExternalBackend', function describeF() { beforeEach(done => { this.clock = sinon.useFakeTimers({ shouldAdvanceTime: true }); const clients = getClients(true); - return checkExternalBackend(clients, awsLocations, 'aws_s3', false, - externalBackendHealthCheckInterval, done); + return checkExternalBackend(clients, awsLocations, 'aws_s3', false, externalBackendHealthCheckInterval, done); }); afterEach(() => { this.clock.restore(); }); - it('should not refresh response before externalBackendHealthCheckInterval', - done => { + it('should not refresh response before externalBackendHealthCheckInterval', done => { const clients = getClients(false); - return checkExternalBackend(clients, awsLocations, 'aws_s3', - false, externalBackendHealthCheckInterval, (err, res) => { - if (err) { - return done(err); - } - assert.strictEqual(res[0].awsbackend, statusSuccess); - return done(); - }); - }); - - it('should refresh response after externalBackendHealthCheckInterval', - done => { - const clients = getClients(false); - setTimeout(() => { - checkExternalBackend(clients, awsLocations, 'aws_s3', - false, externalBackendHealthCheckInterval, (err, res) => { + return checkExternalBackend( + clients, + awsLocations, + 'aws_s3', + false, + externalBackendHealthCheckInterval, + (err, res) => { if (err) { return done(err); } - assert.strictEqual(res[0].awsbackend, statusFailure); + assert.strictEqual(res[0].awsbackend, statusSuccess); return done(); - }); + }, + ); + }); + + it('should refresh response after externalBackendHealthCheckInterval', done => { + const clients = getClients(false); + setTimeout(() => { + checkExternalBackend( + clients, + awsLocations, + 'aws_s3', + false, + externalBackendHealthCheckInterval, + (err, res) => { + if (err) { + return done(err); + } + assert.strictEqual(res[0].awsbackend, statusFailure); + return done(); + }, + ); }, externalBackendHealthCheckInterval + 1); this.clock.next(); // test faster }); diff --git a/tests/unit/utils/pushReplicationMetric.js b/tests/unit/utils/pushReplicationMetric.js index 02b9c944e6..7e818f1d77 100644 --- a/tests/unit/utils/pushReplicationMetric.js +++ b/tests/unit/utils/pushReplicationMetric.js @@ -1,31 +1,25 @@ const assert = require('assert'); const { ObjectMD } = require('arsenal').models; -const { getMetricToPush } = - require('../../../lib/routes/utilities/pushReplicationMetric'); +const { getMetricToPush } = require('../../../lib/routes/utilities/pushReplicationMetric'); describe('getMetricToPush', () => { it('should push metrics when putting a new replica version', () => { - const prevObjectMD = new ObjectMD() - .setVersionId('1'); - const objectMD = new ObjectMD() - .setVersionId('2') - .setReplicationStatus('REPLICA'); + const prevObjectMD = new ObjectMD().setVersionId('1'); + const objectMD = new ObjectMD().setVersionId('2').setReplicationStatus('REPLICA'); const result = getMetricToPush(prevObjectMD, objectMD); assert.strictEqual(result, 'replicateObject'); }); it('should not push metrics for non-replica operations', () => { const prevObjectMD = new ObjectMD(); - const objectMD = new ObjectMD() - .setReplicationStatus('COMPLETED'); + const objectMD = new ObjectMD().setReplicationStatus('COMPLETED'); const result = getMetricToPush(prevObjectMD, objectMD); assert.strictEqual(result, null); }); it('should push metrics for replica operations with tagging', () => { - const prevObjectMD = new ObjectMD() - .setVersionId('1'); + const prevObjectMD = new ObjectMD().setVersionId('1'); const objectMD = new ObjectMD() .setVersionId('1') .setReplicationStatus('REPLICA') @@ -34,80 +28,53 @@ describe('getMetricToPush', () => { assert.strictEqual(result, 'replicateTags'); }); - it('should push metrics for replica operations when deleting tagging', - () => { - const prevObjectMD = new ObjectMD() - .setTags({ 'object-tag-key': 'object-tag-value' }); + it('should push metrics for replica operations when deleting tagging', () => { + const prevObjectMD = new ObjectMD().setTags({ 'object-tag-key': 'object-tag-value' }); const objectMD = new ObjectMD().setReplicationStatus('REPLICA'); const result = getMetricToPush(prevObjectMD, objectMD); assert.strictEqual(result, 'replicateTags'); }); - it('should not push metrics for replica operations with tagging ' + - 'if tags are equal', - () => { - const prevObjectMD = new ObjectMD() - .setVersionId('1') - .setTags({ 'object-tag-key': 'object-tag-value' }); - const objectMD = new ObjectMD() - .setVersionId('1') - .setReplicationStatus('REPLICA') - .setTags({ 'object-tag-key': 'object-tag-value' }); - const result = getMetricToPush(prevObjectMD, objectMD); - assert.strictEqual(result, null); - } - ); + it('should not push metrics for replica operations with tagging ' + 'if tags are equal', () => { + const prevObjectMD = new ObjectMD().setVersionId('1').setTags({ 'object-tag-key': 'object-tag-value' }); + const objectMD = new ObjectMD() + .setVersionId('1') + .setReplicationStatus('REPLICA') + .setTags({ 'object-tag-key': 'object-tag-value' }); + const result = getMetricToPush(prevObjectMD, objectMD); + assert.strictEqual(result, null); + }); it('should push metrics for replica operations with acl', () => { - const prevObjectMD = new ObjectMD() - .setVersionId('1'); + const prevObjectMD = new ObjectMD().setVersionId('1'); const objectMD = new ObjectMD(); const publicACL = objectMD.getAcl(); publicACL.Canned = 'public-read'; - objectMD - .setReplicationStatus('REPLICA') - .setAcl(publicACL) - .setVersionId('1'); + objectMD.setReplicationStatus('REPLICA').setAcl(publicACL).setVersionId('1'); const result = getMetricToPush(prevObjectMD, objectMD); assert.strictEqual(result, 'replicateTags'); }); - - it('should push metrics for replica operations when resetting acl', - () => { + it('should push metrics for replica operations when resetting acl', () => { const prevObjectMD = new ObjectMD(); const publicACL = prevObjectMD.getAcl(); publicACL.Canned = 'public-read'; - prevObjectMD - .setReplicationStatus('REPLICA') - .setAcl(publicACL) - .setVersionId('1'); + prevObjectMD.setReplicationStatus('REPLICA').setAcl(publicACL).setVersionId('1'); const objectMD = new ObjectMD(); const privateACL = objectMD.getAcl(); privateACL.Canned = 'private'; - objectMD - .setReplicationStatus('REPLICA') - .setAcl(privateACL) - .setVersionId('1'); + objectMD.setReplicationStatus('REPLICA').setAcl(privateACL).setVersionId('1'); const result = getMetricToPush(prevObjectMD, objectMD); assert.strictEqual(result, 'replicateTags'); }); - it('should not push metrics for replica operations with acl ' + - 'when they are equal', - () => { - const objectMD = new ObjectMD(); - const publicACL = objectMD.getAcl(); - publicACL.Canned = 'public-read'; - objectMD - .setReplicationStatus('REPLICA') - .setAcl(publicACL) - .setVersionId('1'); - const prevObjectMD = new ObjectMD() - .setAcl(publicACL) - .setVersionId('1'); - const result = getMetricToPush(prevObjectMD, objectMD); - assert.strictEqual(result, null); - } - ); + it('should not push metrics for replica operations with acl ' + 'when they are equal', () => { + const objectMD = new ObjectMD(); + const publicACL = objectMD.getAcl(); + publicACL.Canned = 'public-read'; + objectMD.setReplicationStatus('REPLICA').setAcl(publicACL).setVersionId('1'); + const prevObjectMD = new ObjectMD().setAcl(publicACL).setVersionId('1'); + const result = getMetricToPush(prevObjectMD, objectMD); + assert.strictEqual(result, null); + }); }); diff --git a/tests/unit/utils/request.js b/tests/unit/utils/request.js index d403660e27..0ddcc4dfe9 100644 --- a/tests/unit/utils/request.js +++ b/tests/unit/utils/request.js @@ -102,10 +102,7 @@ function testHandler(req, res) { case '/raw': return respondWithValue(req, res, ['bitsandbytes']); case '/json': - return respondWithValue(req, res, [ - postJsonStringified.slice(0, 3), - postJsonStringified.slice(3) - ]); + return respondWithValue(req, res, [postJsonStringified.slice(0, 3), postJsonStringified.slice(3)]); case '/post': if (req.method !== 'POST') { return respondWithError(req, res, 405); @@ -124,7 +121,7 @@ function testHandler(req, res) { } function createProxyServer(proto, targetHost, hostname, port, callback) { - const target = new URL(targetHost); + const target = new URL(targetHost); let options = {}; let serverType = http; if (proto === 'https') { @@ -138,10 +135,7 @@ function createProxyServer(proto, targetHost, hostname, port, callback) { proxy.on('connect', (req, clnt) => { const svr = net.connect(target.port, target.hostname, () => { // handle http -> https - clnt.write( - `HTTP/${req.httpVersion} 200 Connection Established\r\n` + - '\r\n' - ); + clnt.write(`HTTP/${req.httpVersion} 200 Connection Established\r\n` + '\r\n'); svr.pipe(clnt); clnt.pipe(svr); }); @@ -157,8 +151,7 @@ function createTestServer(proto, hostname, port, handler, callback) { options = { key: testKey, cert: testCert }; serverType = https; } - const server = serverType.createServer(options, - handler); + const server = serverType.createServer(options, handler); server.on('error', err => { process.stdout.write(`https server: ${err.stack}\n`); process.exit(1); @@ -167,10 +160,7 @@ function createTestServer(proto, hostname, port, handler, callback) { return server; } -[ - 'http', - 'https', -].forEach(protocol => { +['http', 'https'].forEach(protocol => { describe(`test against ${protocol} server`, () => { const hostname = 'localhost'; const testPort = 4242; @@ -185,37 +175,45 @@ function createTestServer(proto, hostname, port, handler, callback) { before(done => { process.env.NODE_TLS_REJECT_UNAUTHORIZED = 0; - async.series([ - next => { - server = createTestServer( - protocol, hostname, testPort, testHandler, next); - }, - next => { - proxyTarget = createTestServer(protocol, hostname, 8081, - (req, res) => res.end('proxyTarget'), next); - }, - next => { - proxyServer = createProxyServer('http', - targetHost, hostname, proxyPort, next); - }, - next => { - sproxyServer = createProxyServer('https', - targetHost, hostname, sproxyPort, next); - }, - ], done); + async.series( + [ + next => { + server = createTestServer(protocol, hostname, testPort, testHandler, next); + }, + next => { + proxyTarget = createTestServer( + protocol, + hostname, + 8081, + (req, res) => res.end('proxyTarget'), + next, + ); + }, + next => { + proxyServer = createProxyServer('http', targetHost, hostname, proxyPort, next); + }, + next => { + sproxyServer = createProxyServer('https', targetHost, hostname, sproxyPort, next); + }, + ], + done, + ); }); after(done => { process.env.NODE_TLS_REJECT_UNAUTHORIZED = 1; - async.series([ - next => server.close(next), - next => proxyTarget.close(next), - next => { - proxyServer.close(); - sproxyServer.close(); - next(); - }, - ], done); + async.series( + [ + next => server.close(next), + next => proxyTarget.close(next), + next => { + proxyServer.close(); + sproxyServer.close(); + next(); + }, + ], + done, + ); }); afterEach(() => { @@ -251,35 +249,34 @@ function createTestServer(proto, hostname, port, handler, callback) { }); it('should return data', done => { - request.request(`${host}/raw`, { json: false }, - (err, res, body) => { - assert.ifError(err); - assert.equal(body, 'bitsandbytes'); - done(); - }); + request.request(`${host}/raw`, { json: false }, (err, res, body) => { + assert.ifError(err); + assert.equal(body, 'bitsandbytes'); + done(); + }); }); it('should convert output to json if "json" flag is set', done => { - request.request(`${host}/json`, { json: true }, - (err, res, body) => { - assert.ifError(err); - assert.deepStrictEqual(body, postJson); - done(); - }); + request.request(`${host}/json`, { json: true }, (err, res, body) => { + assert.ifError(err); + assert.deepStrictEqual(body, postJson); + done(); + }); }); it('should set method to GET if it is missing', done => { - const req = request.request(`${host}`, - (err, res) => { - assert.ifError(err); - assert.equal(res.statusCode, 200); - assert.equal(req.method, 'GET'); - done(); - }); + const req = request.request(`${host}`, (err, res) => { + assert.ifError(err); + assert.equal(res.statusCode, 200); + assert.equal(req.method, 'GET'); + done(); + }); }); it('should set headers', done => { - const req = request.request(`${host}`, { + const req = request.request( + `${host}`, + { headers: { 'TEST-HEADERS-ONE': 'test-value-one', 'TEST-HEADERS-TWO': 'test-value-two', @@ -293,75 +290,74 @@ function createTestServer(proto, hostname, port, handler, callback) { 'test-headers-two': 'test-value-two', }); done(); - }); + }, + ); }); }); describe('post', () => { it('should post data', done => { - request.post(`${host}/post`, { body: postData }, - (err, res, body) => { - assert.ifError(err); - assert.equal(res.statusCode, 200); - assert.equal(body, '{"body": "post completed"}'); - done(); - }); + request.post(`${host}/post`, { body: postData }, (err, res, body) => { + assert.ifError(err); + assert.equal(res.statusCode, 200); + assert.equal(body, '{"body": "post completed"}'); + done(); + }); }); it('should post with json data', done => { - request.post(`${host}/postjson`, { body: { key: 'value' } }, - (err, res, body) => { - assert.ifError(err); - assert.equal(res.statusCode, 200); - assert.equal(body, '{"body": "post completed"}'); - done(); - }); + request.post(`${host}/postjson`, { body: { key: 'value' } }, (err, res, body) => { + assert.ifError(err); + assert.equal(res.statusCode, 200); + assert.equal(body, '{"body": "post completed"}'); + done(); + }); }); it('should post with empty body', done => { - request.post(`${host}/postempty`, - (err, res, body) => { - assert.ifError(err); - assert.equal(res.statusCode, 200); - assert.equal(body, '{"body": "post completed"}'); - done(); - }); + request.post(`${host}/postempty`, (err, res, body) => { + assert.ifError(err); + assert.equal(res.statusCode, 200); + assert.equal(body, '{"body": "post completed"}'); + done(); + }); }); it('should post with json data (json response)', done => { - request.post(`${host}/postjson`, - { body: { key: 'value' }, json: true }, - (err, res, body) => { - assert.ifError(err); - assert.equal(res.statusCode, 200); - assert.deepStrictEqual(body, { - body: 'post completed', - }); - done(); + request.post(`${host}/postjson`, { body: { key: 'value' }, json: true }, (err, res, body) => { + assert.ifError(err); + assert.equal(res.statusCode, 200); + assert.deepStrictEqual(body, { + body: 'post completed', }); + done(); + }); }); it('should set content-type JSON if missing', done => { - const req = request.post(`${host}`, { + const req = request.post( + `${host}`, + { body: postJson, - headers: { 'EXTRA': 'header' }, + headers: { EXTRA: 'header' }, }, (err, res) => { assert.ifError(err); assert.equal(res.statusCode, 200); checkForHeaders(req.getHeaders(), { 'content-type': 'application/json', - 'content-length': - Buffer.byteLength(postJsonStringified), - 'extra': 'header', + 'content-length': Buffer.byteLength(postJsonStringified), + extra: 'header', }); done(); - }); + }, + ); }); - it('should not overwrite existing content-type header value', - done => { - const req = request.post(`${host}`, { + it('should not overwrite existing content-type header value', done => { + const req = request.post( + `${host}`, + { body: postJson, headers: { 'Content-Type': 'text/plain' }, }, @@ -370,22 +366,22 @@ function createTestServer(proto, hostname, port, handler, callback) { assert.equal(res.statusCode, 200); checkForHeaders(req.getHeaders(), { 'content-type': 'text/plain', - 'content-length': - Buffer.byteLength(postJsonStringified), + 'content-length': Buffer.byteLength(postJsonStringified), }); done(); - }); - }); + }, + ); + }); }); }); }); describe('utilities::request error handling', () => { - it('should throw an error if arguments are missing', () => { + it('should throw an error if arguments are missing', () => { assert.throws(request.request); }); - it('should throw an error if callback argument is missing', () => { + it('should throw an error if callback argument is missing', () => { assert.throws(() => request.request('http://test')); }); @@ -427,7 +423,7 @@ describe('utilities::createHeaders', () => { { 'content-type': 'test/one', 'content-length': 1, - } + }, ); }); }); diff --git a/tests/unit/utils/responseStreamData.js b/tests/unit/utils/responseStreamData.js index 10066bfc42..f21861955d 100644 --- a/tests/unit/utils/responseStreamData.js +++ b/tests/unit/utils/responseStreamData.js @@ -8,8 +8,7 @@ const { config } = require('../../../lib/Config'); const { client, implName, data } = require('../../../lib/data/wrapper'); const kms = require('../../../lib/kms/wrapper'); const vault = require('../../../lib/auth/vault'); -const locationStorageCheck = - require('../../../lib/api/apiUtils/object/locationStorageCheck'); +const locationStorageCheck = require('../../../lib/api/apiUtils/object/locationStorageCheck'); const metadata = require('../../../lib/metadata/wrapper'); const routesUtils = s3routes.routesUtils; @@ -51,10 +50,12 @@ describe.skip('responseStreamData:', () => { it('should stream full requested object data for one part object', done => { ds.push(null, dataStoreEntry); - const dataLocations = [{ - key: 1, - dataStore: 'mem', - }]; + const dataLocations = [ + { + key: 1, + dataStore: 'mem', + }, + ]; const response = httpMocks.createResponse({ eventEmitter: EventEmitter, }); @@ -63,8 +64,16 @@ describe.skip('responseStreamData:', () => { assert.strictEqual(data, postBody.toString()); done(); }); - return responseStreamData(errCode, overrideHeaders, resHeaders, - dataLocations, dataRetrievalParams, response, null, log); + return responseStreamData( + errCode, + overrideHeaders, + resHeaders, + dataLocations, + dataRetrievalParams, + response, + null, + log, + ); }); it('should stream full requested object data for two part object', done => { @@ -81,7 +90,8 @@ describe.skip('responseStreamData:', () => { dataStore: 'mem', start: 11, size: 11, - }]; + }, + ]; const response = httpMocks.createResponse({ eventEmitter: EventEmitter, }); @@ -91,17 +101,27 @@ describe.skip('responseStreamData:', () => { assert.strictEqual(data, doublePostBody); done(); }); - return responseStreamData(errCode, overrideHeaders, resHeaders, - dataLocations, dataRetrievalParams, response, null, log); + return responseStreamData( + errCode, + overrideHeaders, + resHeaders, + dataLocations, + dataRetrievalParams, + response, + null, + log, + ); }); it('#334 non-regression test, destroy connection on error', done => { - const dataLocations = [{ - key: 1, - dataStore: 'mem', - start: 0, - size: 11, - }]; + const dataLocations = [ + { + key: 1, + dataStore: 'mem', + start: 0, + size: 11, + }, + ]; const prev = data.get; data.get = (objectGetInfo, response, log, cb) => { setTimeout(() => cb(errors.InternalError), 1000); @@ -117,12 +137,19 @@ describe.skip('responseStreamData:', () => { response.on('end', () => { data.get = prev; if (!destroyed) { - return done(new Error('end reached instead of destroying ' + - 'connection')); + return done(new Error('end reached instead of destroying ' + 'connection')); } return done(); }); - return responseStreamData(errCode, overrideHeaders, resHeaders, - dataLocations, dataRetrievalParams, response, null, log); + return responseStreamData( + errCode, + overrideHeaders, + resHeaders, + dataLocations, + dataRetrievalParams, + response, + null, + log, + ); }); }); diff --git a/tests/unit/utils/serverAccessLogger.js b/tests/unit/utils/serverAccessLogger.js index 13cc1ae68e..648b204d6a 100644 --- a/tests/unit/utils/serverAccessLogger.js +++ b/tests/unit/utils/serverAccessLogger.js @@ -477,7 +477,7 @@ describe('serverAccessLogger utility functions', () => { it('should return Content-Length from response for objectGet', () => { const request = { apiMethod: 'objectGet' }; const response = { - getHeader: name => name === 'Content-Length' ? '12345' : null, + getHeader: name => (name === 'Content-Length' ? '12345' : null), }; const result = getObjectSize(request, response); assert.strictEqual(result, 12345); @@ -510,7 +510,7 @@ describe('serverAccessLogger utility functions', () => { it('should handle Content-Length of 0 for objectGet', () => { const request = { apiMethod: 'objectGet' }; const response = { - getHeader: name => name === 'Content-Length' ? '0' : null, + getHeader: name => (name === 'Content-Length' ? '0' : null), }; const result = getObjectSize(request, response); assert.strictEqual(result, 0); @@ -519,7 +519,7 @@ describe('serverAccessLogger utility functions', () => { it('should handle Content-Length of number 0 for objectGet', () => { const request = { apiMethod: 'objectGet' }; const response = { - getHeader: name => name === 'Content-Length' ? 0 : null, + getHeader: name => (name === 'Content-Length' ? 0 : null), }; const result = getObjectSize(request, response); assert.strictEqual(result, 0); @@ -604,7 +604,7 @@ describe('serverAccessLogger utility functions', () => { it('should return Content-Length from response when bytesSent is not provided', () => { const res = { - getHeader: name => name === 'Content-Length' ? '67890' : null, + getHeader: name => (name === 'Content-Length' ? '67890' : null), }; const result = getBytesSent(res, null); assert.strictEqual(result, '67890'); @@ -625,7 +625,7 @@ describe('serverAccessLogger utility functions', () => { it('should prefer bytesSent over Content-Length', () => { const res = { - getHeader: name => name === 'Content-Length' ? '99999' : null, + getHeader: name => (name === 'Content-Length' ? '99999' : null), }; const bytesSent = 11111; const result = getBytesSent(res, bytesSent); @@ -634,7 +634,7 @@ describe('serverAccessLogger utility functions', () => { it('should handle bytesSent as 0', () => { const res = { - getHeader: name => name === 'Content-Length' ? '99999' : null, + getHeader: name => (name === 'Content-Length' ? '99999' : null), }; const bytesSent = 0; const result = getBytesSent(res, bytesSent); @@ -643,7 +643,7 @@ describe('serverAccessLogger utility functions', () => { it('should handle Content-Length as number 0', () => { const res = { - getHeader: name => name === 'Content-Length' ? 0 : null, + getHeader: name => (name === 'Content-Length' ? 0 : null), }; const result = getBytesSent(res, null); assert.strictEqual(result, 0); @@ -651,7 +651,7 @@ describe('serverAccessLogger utility functions', () => { it('should handle Content-Length as string "0"', () => { const res = { - getHeader: name => name === 'Content-Length' ? '0' : null, + getHeader: name => (name === 'Content-Length' ? '0' : null), }; const result = getBytesSent(res, null); assert.strictEqual(result, '0'); @@ -893,10 +893,10 @@ describe('serverAccessLogger utility functions', () => { analyticsAccountName: 'testAccount', analyticsUserName: 'testUser', analyticsBytesDeleted: 0, - startTime: 1000000000n, - onFinishEndTime: 1009000000n, + startTime: 1000000000n, + onFinishEndTime: 1009000000n, startTurnAroundTime: 1003000000n, - onCloseEndTime: 1020500000n, + onCloseEndTime: 1020500000n, startTimeUnixMS: 1234567890000, bucketOwner: 'bucketOwner123', bucketName: 'test-bucket', @@ -915,8 +915,8 @@ describe('serverAccessLogger utility functions', () => { headers: { 'content-length': '1024', 'user-agent': 'aws-cli/2.0.0', - 'referer': 'https://example.com', - 'host': 's3.amazonaws.com', + referer: 'https://example.com', + host: 's3.amazonaws.com', 'x-forwarded-for': '192.168.1.100', }, parsedContentLength: 1024, @@ -938,7 +938,9 @@ describe('serverAccessLogger utility functions', () => { }, statusCode: 200, getHeader: name => { - if (name === 'Content-Length') {return '2048';} + if (name === 'Content-Length') { + return '2048'; + } return null; }, }; @@ -1213,7 +1215,7 @@ describe('serverAccessLogger utility functions', () => { assert.strictEqual(mockLogger.write.callCount, 1); const writtenData = mockLogger.write.firstCall.args[0]; assert.strictEqual(writtenData.endsWith('\n'), true); - + // Should be valid JSON without the newline const jsonData = writtenData.trim(); assert.doesNotThrow(() => JSON.parse(jsonData)); @@ -1437,7 +1439,7 @@ describe('serverAccessLogger utility functions', () => { assert.strictEqual(mockLogger.write.callCount, 1); const loggedData = JSON.parse(mockLogger.write.firstCall.args[0].trim()); - + // Verify operation is REST.GET.BACKBEAT assert.strictEqual(loggedData.operation, 'REST.GET.BACKBEAT'); }); @@ -1467,7 +1469,7 @@ describe('serverAccessLogger utility functions', () => { assert.strictEqual(mockLogger.write.callCount, 1); const loggedData = JSON.parse(mockLogger.write.firstCall.args[0].trim()); - + // Verify loggingEnabled is false (overridden by backbeat) assert.strictEqual(loggedData.loggingEnabled, false); // But TargetBucket and TargetPrefix should still be logged @@ -1760,8 +1762,7 @@ describe('serverAccessLogger utility functions', () => { getCanonicalID: () => 'replication-canonical-id', isRequesterPublicUser: () => false, isRequesterAnIAMUser: () => false, - getArn: () => - 'arn:aws:sts::123456789012:assumed-role/replication-role/backbeat-replication', + getArn: () => 'arn:aws:sts::123456789012:assumed-role/replication-role/backbeat-replication', getAuthVersion: () => 'AWS4-HMAC-SHA256', getAuthType: () => 'REST-HEADER', getAccessKey: () => 'replication-access-key', @@ -1832,13 +1833,14 @@ describe('serverAccessLogger utility functions', () => { assert.strictEqual(loggedData.objectKey, 'replicated.txt'); // Requester is the assumed-role ARN from auth - assert.strictEqual(loggedData.requester, - 'arn:aws:sts::123456789012:assumed-role/replication-role/backbeat-replication'); + assert.strictEqual( + loggedData.requester, + 'arn:aws:sts::123456789012:assumed-role/replication-role/backbeat-replication', + ); // requestURI is synthesized to look like a normal S3 PUT, // not the internal /_/backbeat/data path - assert.strictEqual(loggedData.requestURI, - 'PUT /dest-bucket/replicated.txt HTTP/1.1'); + assert.strictEqual(loggedData.requestURI, 'PUT /dest-bucket/replicated.txt HTTP/1.1'); // HTTP-layer fields that AWS blanks for replication assert.strictEqual(loggedData.clientIP, undefined); @@ -1861,8 +1863,7 @@ describe('serverAccessLogger utility functions', () => { getCanonicalID: () => 'replication-canonical-id', isRequesterPublicUser: () => false, isRequesterAnIAMUser: () => false, - getArn: () => - 'arn:aws:sts::123456789012:assumed-role/replication-role/backbeat-replication', + getArn: () => 'arn:aws:sts::123456789012:assumed-role/replication-role/backbeat-replication', getAuthVersion: () => 'AWS4-HMAC-SHA256', getAuthType: () => 'REST-HEADER', getAccessKey: () => 'replication-access-key', @@ -1910,8 +1911,7 @@ describe('serverAccessLogger utility functions', () => { assert.strictEqual(loggedData.operation, 'REST.DELETE.OBJECT'); assert.strictEqual(loggedData.loggingEnabled, true); - assert.strictEqual(loggedData.requestURI, - 'DELETE /dest-bucket/replicated.txt HTTP/1.1'); + assert.strictEqual(loggedData.requestURI, 'DELETE /dest-bucket/replicated.txt HTTP/1.1'); // HTTP-layer fields that AWS blanks for replication assert.strictEqual(loggedData.clientIP, undefined); @@ -1919,8 +1919,10 @@ describe('serverAccessLogger utility functions', () => { assert.strictEqual(loggedData.referer, undefined); // Replication identity preserved - assert.strictEqual(loggedData.requester, - 'arn:aws:sts::123456789012:assumed-role/replication-role/backbeat-replication'); + assert.strictEqual( + loggedData.requester, + 'arn:aws:sts::123456789012:assumed-role/replication-role/backbeat-replication', + ); }); it('should produce a REST.PUT.OBJECT_TAGGING entry for tag-only replication', () => { @@ -1930,8 +1932,7 @@ describe('serverAccessLogger utility functions', () => { getCanonicalID: () => 'replication-canonical-id', isRequesterPublicUser: () => false, isRequesterAnIAMUser: () => false, - getArn: () => - 'arn:aws:sts::123456789012:assumed-role/replication-role/backbeat-replication', + getArn: () => 'arn:aws:sts::123456789012:assumed-role/replication-role/backbeat-replication', getAuthVersion: () => 'AWS4-HMAC-SHA256', getAuthType: () => 'REST-HEADER', getAccessKey: () => 'replication-access-key', @@ -1981,8 +1982,10 @@ describe('serverAccessLogger utility functions', () => { assert.strictEqual(loggedData.operation, 'REST.PUT.OBJECT_TAGGING'); assert.strictEqual(loggedData.loggingEnabled, true); - assert.strictEqual(loggedData.requestURI, - `PUT /dest-bucket/replicated.txt?tagging&versionId=${versionId} HTTP/1.1`); + assert.strictEqual( + loggedData.requestURI, + `PUT /dest-bucket/replicated.txt?tagging&versionId=${versionId} HTTP/1.1`, + ); assert.strictEqual(loggedData.versionId, versionId); // HTTP-layer fields that AWS blanks for replication @@ -1991,8 +1994,10 @@ describe('serverAccessLogger utility functions', () => { assert.strictEqual(loggedData.referer, undefined); // Replication identity preserved - assert.strictEqual(loggedData.requester, - 'arn:aws:sts::123456789012:assumed-role/replication-role/backbeat-replication'); + assert.strictEqual( + loggedData.requester, + 'arn:aws:sts::123456789012:assumed-role/replication-role/backbeat-replication', + ); }); it('should produce a REST.PUT.ACL entry for ACL-only replication', () => { @@ -2002,8 +2007,7 @@ describe('serverAccessLogger utility functions', () => { getCanonicalID: () => 'replication-canonical-id', isRequesterPublicUser: () => false, isRequesterAnIAMUser: () => false, - getArn: () => - 'arn:aws:sts::123456789012:assumed-role/replication-role/backbeat-replication', + getArn: () => 'arn:aws:sts::123456789012:assumed-role/replication-role/backbeat-replication', getAuthVersion: () => 'AWS4-HMAC-SHA256', getAuthType: () => 'REST-HEADER', getAccessKey: () => 'replication-access-key', @@ -2054,8 +2058,10 @@ describe('serverAccessLogger utility functions', () => { assert.strictEqual(loggedData.operation, 'REST.PUT.ACL'); assert.strictEqual(loggedData.loggingEnabled, true); - assert.strictEqual(loggedData.requestURI, - `PUT /dest-bucket/replicated.txt?acl&versionId=${versionId} HTTP/1.1`); + assert.strictEqual( + loggedData.requestURI, + `PUT /dest-bucket/replicated.txt?acl&versionId=${versionId} HTTP/1.1`, + ); assert.strictEqual(loggedData.versionId, versionId); assert.strictEqual(loggedData.aclRequired, 'Yes'); @@ -2065,8 +2071,10 @@ describe('serverAccessLogger utility functions', () => { assert.strictEqual(loggedData.referer, undefined); // Replication identity preserved - assert.strictEqual(loggedData.requester, - 'arn:aws:sts::123456789012:assumed-role/replication-role/backbeat-replication'); + assert.strictEqual( + loggedData.requester, + 'arn:aws:sts::123456789012:assumed-role/replication-role/backbeat-replication', + ); }); it('should log replication with error code on failure', () => { @@ -2109,7 +2117,5 @@ describe('serverAccessLogger utility functions', () => { assert.strictEqual(loggedData.errorCode, 'InternalError'); assert.strictEqual(loggedData.httpCode, 500); }); - }); }); - diff --git a/tests/unit/utils/setPartRanges.js b/tests/unit/utils/setPartRanges.js index 52c261e4e1..ef56b3e1e1 100644 --- a/tests/unit/utils/setPartRanges.js +++ b/tests/unit/utils/setPartRanges.js @@ -6,76 +6,85 @@ describe('setPartRanges function', () => { it('should set range on a one part object', () => { const dataLocations = [{ key: '1' }]; const outerRange = [2, 8]; - const actual = - setPartRanges(dataLocations, outerRange); - assert.deepStrictEqual(actual, [{ - key: '1', - range: [2, 8], - }]); + const actual = setPartRanges(dataLocations, outerRange); + assert.deepStrictEqual(actual, [ + { + key: '1', + range: [2, 8], + }, + ]); }); - it('for a 3-part object, should include full first part, set range on ' + - 'middle part and exclude last part if range request starts at 0' + - 'and ends in the middle of the second part', + it( + 'for a 3-part object, should include full first part, set range on ' + + 'middle part and exclude last part if range request starts at 0' + + 'and ends in the middle of the second part', () => { - const dataLocations = [{ key: '1', size: '4', start: '0' }, + const dataLocations = [ + { key: '1', size: '4', start: '0' }, { key: '2', size: '10', start: '4' }, { key: '3', size: '20', start: '14' }, ]; const outerRange = [0, 10]; - const actual = - setPartRanges(dataLocations, outerRange); - assert.deepStrictEqual(actual, [{ key: '1', size: '4', start: '0' }, - { key: '2', size: '7', start: '4', range: [0, 6] }]); - }); + const actual = setPartRanges(dataLocations, outerRange); + assert.deepStrictEqual(actual, [ + { key: '1', size: '4', start: '0' }, + { key: '2', size: '7', start: '4', range: [0, 6] }, + ]); + }, + ); - it('for a 3-part object, should include part of first part, all of ' + - 'second part and part of third part if range request starts within ' + - 'first part and ends before end of last part', + it( + 'for a 3-part object, should include part of first part, all of ' + + 'second part and part of third part if range request starts within ' + + 'first part and ends before end of last part', () => { - const dataLocations = [{ key: '1', size: '4', start: '0' }, + const dataLocations = [ + { key: '1', size: '4', start: '0' }, { key: '2', size: '10', start: '4' }, { key: '3', size: '20', start: '14' }, ]; const outerRange = [2, 18]; - const actual = - setPartRanges(dataLocations, outerRange); - assert.deepStrictEqual(actual, [{ key: '1', size: '2', start: '0', - range: [2, 3] }, - { key: '2', size: '10', start: '4' }, - { key: '3', size: '5', start: '14', range: [0, 4] }, + const actual = setPartRanges(dataLocations, outerRange); + assert.deepStrictEqual(actual, [ + { key: '1', size: '2', start: '0', range: [2, 3] }, + { key: '2', size: '10', start: '4' }, + { key: '3', size: '5', start: '14', range: [0, 4] }, ]); - }); + }, + ); - it('for a 3-part object, should include only a range of the middle part ' + - 'if the range excludes both the beginning and the end', + it( + 'for a 3-part object, should include only a range of the middle part ' + + 'if the range excludes both the beginning and the end', () => { - const dataLocations = [{ key: '1', size: '4', start: '0' }, + const dataLocations = [ + { key: '1', size: '4', start: '0' }, { key: '2', size: '10', start: '4' }, { key: '3', size: '20', start: '14' }, ]; const outerRange = [5, 7]; - const actual = - setPartRanges(dataLocations, outerRange); - assert.deepStrictEqual(actual, [{ key: '2', size: '3', start: '4', - range: [1, 3] }, - ]); - }); + const actual = setPartRanges(dataLocations, outerRange); + assert.deepStrictEqual(actual, [{ key: '2', size: '3', start: '4', range: [1, 3] }]); + }, + ); - it('for a 3-part object, should include only a range of the middle part ' + - 'and all of the third part if the range excludes a portion of the ' + - 'beginning', + it( + 'for a 3-part object, should include only a range of the middle part ' + + 'and all of the third part if the range excludes a portion of the ' + + 'beginning', () => { - const dataLocations = [{ key: '1', size: '4', start: '0' }, + const dataLocations = [ + { key: '1', size: '4', start: '0' }, { key: '2', size: '10', start: '4' }, { key: '3', size: '20', start: '14' }, ]; const outerRange = [5, 34]; - const actual = - setPartRanges(dataLocations, outerRange); - assert.deepStrictEqual(actual, [{ key: '2', size: '9', start: '4', - range: [1, 9] }, + const actual = setPartRanges(dataLocations, outerRange); + assert.deepStrictEqual(actual, [ + { key: '2', size: '9', start: '4', range: [1, 9] }, { key: '3', size: '20', start: '14' }, ]); - }); + }, + ); }); diff --git a/tests/unit/utils/validateSearch.js b/tests/unit/utils/validateSearch.js index 4b803278aa..47dc0a1485 100644 --- a/tests/unit/utils/validateSearch.js +++ b/tests/unit/utils/validateSearch.js @@ -1,8 +1,6 @@ const assert = require('assert'); const { errorInstances } = require('arsenal'); -const validateSearch = - require('../../../lib/api/apiUtils/bucket/validateSearch'); - +const validateSearch = require('../../../lib/api/apiUtils/bucket/validateSearch'); describe('validate search where clause', () => { const tests = [ @@ -12,33 +10,30 @@ describe('validate search where clause', () => { result: undefined, }, { - it: 'should allow a simple search with known ' + - 'column attribute', + it: 'should allow a simple search with known ' + 'column attribute', searchParams: '`content-length`="10"', result: undefined, }, { it: 'should allow valid search with AND', - searchParams: '`x-amz-meta-dog`="labrador" ' + - 'AND `x-amz-meta-age`="5"', + searchParams: '`x-amz-meta-dog`="labrador" ' + 'AND `x-amz-meta-age`="5"', result: undefined, }, { it: 'should allow valid search with OR', - searchParams: '`x-amz-meta-dog`="labrador" ' + - 'OR `x-amz-meta-age`="5"', + searchParams: '`x-amz-meta-dog`="labrador" ' + 'OR `x-amz-meta-age`="5"', result: undefined, }, { it: 'should allow valid search with double AND', - searchParams: '`x-amz-meta-dog`="labrador" ' + - 'AND `x-amz-meta-age`="5" ' + - 'AND `x-amz-meta-whatever`="ok"', + searchParams: + '`x-amz-meta-dog`="labrador" ' + 'AND `x-amz-meta-age`="5" ' + 'AND `x-amz-meta-whatever`="ok"', result: undefined, }, { it: 'should allow valid chained search with tables and columns', - searchParams: '`x-amz-meta-dog`="labrador" ' + + searchParams: + '`x-amz-meta-dog`="labrador" ' + 'AND `x-amz-meta-age`="5" ' + 'AND `content-length`="10"' + 'OR isDeleteMarker="true"' + @@ -47,70 +42,70 @@ describe('validate search where clause', () => { }, { it: 'should allow valid LIKE search', - searchParams: '`x-amz-meta-dog` LIKE "lab%" ' + - 'AND `x-amz-meta-age` LIKE "5%" ' + - 'AND `content-length`="10"', + searchParams: + '`x-amz-meta-dog` LIKE "lab%" ' + 'AND `x-amz-meta-age` LIKE "5%" ' + 'AND `content-length`="10"', result: undefined, }, { it: 'should disallow a LIKE search with invalid attribute', searchParams: '`x-zma-meta-dog` LIKE "labrador"', - result: errorInstances.InvalidArgument.customizeDescription('Search ' + - 'param contains unknown attribute: x-zma-meta-dog'), + result: errorInstances.InvalidArgument.customizeDescription( + 'Search ' + 'param contains unknown attribute: x-zma-meta-dog', + ), }, { it: 'should disallow a simple search with unknown attribute', searchParams: '`x-zma-meta-dog`="labrador"', - result: errorInstances.InvalidArgument.customizeDescription('Search ' + - 'param contains unknown attribute: x-zma-meta-dog'), + result: errorInstances.InvalidArgument.customizeDescription( + 'Search ' + 'param contains unknown attribute: x-zma-meta-dog', + ), }, { - it: 'should disallow a compound search with unknown ' + - 'attribute on right', - searchParams: '`x-amz-meta-dog`="labrador" AND ' + - '`x-zma-meta-dog`="labrador"', - result: errorInstances.InvalidArgument.customizeDescription('Search ' + - 'param contains unknown attribute: x-zma-meta-dog'), + it: 'should disallow a compound search with unknown ' + 'attribute on right', + searchParams: '`x-amz-meta-dog`="labrador" AND ' + '`x-zma-meta-dog`="labrador"', + result: errorInstances.InvalidArgument.customizeDescription( + 'Search ' + 'param contains unknown attribute: x-zma-meta-dog', + ), }, { - it: 'should disallow a compound search with unknown ' + - 'attribute on left', - searchParams: '`x-zma-meta-dog`="labrador" AND ' + - '`x-amz-meta-dog`="labrador"', - result: errorInstances.InvalidArgument.customizeDescription('Search ' + - 'param contains unknown attribute: x-zma-meta-dog'), + it: 'should disallow a compound search with unknown ' + 'attribute on left', + searchParams: '`x-zma-meta-dog`="labrador" AND ' + '`x-amz-meta-dog`="labrador"', + result: errorInstances.InvalidArgument.customizeDescription( + 'Search ' + 'param contains unknown attribute: x-zma-meta-dog', + ), }, { - it: 'should disallow a chained search with one invalid ' + - 'table attribute', - searchParams: '`x-amz-meta-dog`="labrador" ' + - 'AND `x-amz-meta-age`="5" ' + - 'OR `x-zma-meta-whatever`="ok"', - result: errorInstances.InvalidArgument.customizeDescription('Search ' + - 'param contains unknown attribute: x-zma-meta-whatever'), + it: 'should disallow a chained search with one invalid ' + 'table attribute', + searchParams: + '`x-amz-meta-dog`="labrador" ' + 'AND `x-amz-meta-age`="5" ' + 'OR `x-zma-meta-whatever`="ok"', + result: errorInstances.InvalidArgument.customizeDescription( + 'Search ' + 'param contains unknown attribute: x-zma-meta-whatever', + ), }, { - it: 'should disallow a simple search with unknown ' + - 'column attribute', + it: 'should disallow a simple search with unknown ' + 'column attribute', searchParams: 'whatever="labrador"', - result: errorInstances.InvalidArgument.customizeDescription('Search ' + - 'param contains unknown attribute: whatever'), + result: errorInstances.InvalidArgument.customizeDescription( + 'Search ' + 'param contains unknown attribute: whatever', + ), }, { - it: 'should disallow a chained search with one invalid ' + - 'column attribute', - searchParams: '`x-amz-meta-dog`="labrador" ' + + it: 'should disallow a chained search with one invalid ' + 'column attribute', + searchParams: + '`x-amz-meta-dog`="labrador" ' + 'AND `x-amz-meta-age`="5" ' + 'OR madeUp="something"' + 'OR `x-amz-meta-whatever`="ok"', - result: errorInstances.InvalidArgument.customizeDescription('Search ' + - 'param contains unknown attribute: madeUp'), + result: errorInstances.InvalidArgument.customizeDescription( + 'Search ' + 'param contains unknown attribute: madeUp', + ), }, { it: 'should disallow unsupported query operators', searchParams: 'x-amz-meta-dog BETWEEN "labrador"', result: errorInstances.InvalidArgument.customizeDescription( - 'Invalid sql where clause sent as search query'), + 'Invalid sql where clause sent as search query', + ), }, { it: 'should allow a simple search with tag query', @@ -126,8 +121,7 @@ describe('validate search where clause', () => { tests.forEach(test => { it(test.it, () => { - const actualResult = - validateSearch(test.searchParams); + const actualResult = validateSearch(test.searchParams); if (test.result === undefined) { assert(typeof actualResult.ast === 'object'); } else { diff --git a/tests/utapi/awsNodeSdk.js b/tests/utapi/awsNodeSdk.js index c0d1030755..975f3cc6a2 100644 --- a/tests/utapi/awsNodeSdk.js +++ b/tests/utapi/awsNodeSdk.js @@ -26,13 +26,15 @@ function wait(timeoutMs, cb) { } function createBucket(bucket, cb) { - s3Client.send(new CreateBucketCommand({ Bucket: bucket })) + s3Client + .send(new CreateBucketCommand({ Bucket: bucket })) .then(data => cb(null, data)) .catch(cb); } function deleteBucket(bucket, cb) { - s3Client.send(new DeleteBucketCommand({ Bucket: bucket })) + s3Client + .send(new DeleteBucketCommand({ Bucket: bucket })) .then(() => cb(null)) .catch(cb); } @@ -44,13 +46,15 @@ function putObject(bucket, key, size, cb) { Key: key, Body: body, }; - s3Client.send(new PutObjectCommand(params)) + s3Client + .send(new PutObjectCommand(params)) .then(data => cb(null, data)) .catch(cb); } function deleteObject(bucket, key, cb) { - s3Client.send(new DeleteObjectCommand({ Bucket: bucket, Key: key })) + s3Client + .send(new DeleteObjectCommand({ Bucket: bucket, Key: key })) .then(() => cb(null)) .catch(cb); } @@ -62,14 +66,16 @@ function deleteObjects(bucket, keys, cb) { Bucket: bucket, Delete: deleteRequest, }; - s3Client.send(new DeleteObjectsCommand(params)) + s3Client + .send(new DeleteObjectsCommand(params)) .then(() => cb(null)) .catch(cb); } function copyObject(bucket, key, cb) { const params = { Bucket: bucket, CopySource: `${bucket}/${key}`, Key: `${key}-copy` }; - s3Client.send(new CopyObjectCommand(params)) + s3Client + .send(new CopyObjectCommand(params)) .then(() => cb(null)) .catch(cb); } @@ -77,7 +83,8 @@ function copyObject(bucket, key, cb) { function enableVersioning(bucket, enable, cb) { const versioningStatus = { Status: enable ? 'Enabled' : 'Disabled' }; const params = { Bucket: bucket, VersioningConfiguration: versioningStatus }; - s3Client.send(new PutBucketVersioningCommand(params)) + s3Client + .send(new PutBucketVersioningCommand(params)) .then(() => cb(null)) .catch(cb); } @@ -101,24 +108,24 @@ async function removeAllVersions(params, callback) { try { const bucket = params.Bucket; const data = await s3Client.send(new ListObjectVersionsCommand(params)); - + if (data.DeleteMarkers && data.DeleteMarkers.length > 0) { await deleteVersionList(data.DeleteMarkers, bucket); } - + if (data.Versions && data.Versions.length > 0) { await deleteVersionList(data.Versions, bucket); } - + if (data.IsTruncated) { - const nextParams = { - Bucket: bucket, - KeyMarker: data.NextKeyMarker, - VersionIdMarker: data.NextVersionIdMarker + const nextParams = { + Bucket: bucket, + KeyMarker: data.NextKeyMarker, + VersionIdMarker: data.NextVersionIdMarker, }; await removeAllVersions(nextParams); } - + callback(); } catch (error) { callback(error); @@ -130,46 +137,58 @@ function objectMPU(bucket, key, parts, partSize, callback) { let uploadId = null; const partNumbers = Array.from(Array(parts).keys()); const initiateMPUParams = { Bucket: bucket, Key: key }; - async.waterfall([ - next => s3Client.send(new CreateMultipartUploadCommand(initiateMPUParams)) - .then(data => { - uploadId = data.UploadId; - return next(); - }) - .catch(next), - next => - async.mapLimit(partNumbers, 1, (partNumber, callback) => { - const body = Buffer.alloc(partSize); - const uploadPartParams = { + async.waterfall( + [ + next => + s3Client + .send(new CreateMultipartUploadCommand(initiateMPUParams)) + .then(data => { + uploadId = data.UploadId; + return next(); + }) + .catch(next), + next => + async.mapLimit( + partNumbers, + 1, + (partNumber, callback) => { + const body = Buffer.alloc(partSize); + const uploadPartParams = { + Bucket: bucket, + Key: key, + PartNumber: partNumber + 1, + UploadId: uploadId, + Body: body, + }; + s3Client + .send(new UploadPartCommand(uploadPartParams)) + .then(data => callback(null, data.ETag)) + .catch(callback); + }, + (err, results) => { + if (err) { + return next(err); + } + ETags = results; + return next(); + }, + ), + next => { + const completeRequest = { Parts: partNumbers.map(n => ({ ETag: ETags[n], PartNumber: n + 1 })) }; + const params = { Bucket: bucket, Key: key, - PartNumber: partNumber + 1, + MultipartUpload: completeRequest, UploadId: uploadId, - Body: body, }; - s3Client.send(new UploadPartCommand(uploadPartParams)) - .then(data => callback(null, data.ETag)) - .catch(callback); - }, (err, results) => { - if (err) { - return next(err); - } - ETags = results; - return next(); - }), - next => { - const completeRequest = { Parts: partNumbers.map(n => ({ ETag: ETags[n], PartNumber: n + 1 })) }; - const params = { - Bucket: bucket, - Key: key, - MultipartUpload: completeRequest, - UploadId: uploadId, - }; - s3Client.send(new CompleteMultipartUploadCommand(params)) - .then(data => next(null, data)) - .catch(next); - }, - ], callback); + s3Client + .send(new CompleteMultipartUploadCommand(params)) + .then(data => next(null, data)) + .catch(next); + }, + ], + callback, + ); } function removeVersions(buckets, cb) { @@ -177,7 +196,8 @@ function removeVersions(buckets, cb) { } function getObject(bucket, key, cb) { - s3Client.send(new GetObjectCommand({ Bucket: bucket, Key: key })) + s3Client + .send(new GetObjectCommand({ Bucket: bucket, Key: key })) .then(data => cb(null, data)) .catch(cb); } @@ -195,7 +215,7 @@ describe('utapi v2 metrics incoming and outgoing bytes', function t() { } before(() => { - s3Client = new S3Client(getConfig('default')); + s3Client = new S3Client(getConfig('default')); utapi.start(); }); afterEach(() => { @@ -206,18 +226,23 @@ describe('utapi v2 metrics incoming and outgoing bytes', function t() { }); it('should set metrics for createBucket and deleteBucket', done => { const bucket = 'bucket1'; - async.series([ - next => createBucket(bucket, next), - next => wait(WAIT_MS, () => { - checkMetrics(0, 0, 0); - next(); - }), - next => deleteBucket(bucket, next), - next => wait(WAIT_MS, () => { - checkMetrics(0, 0, 0); - next(); - }), - ], done); + async.series( + [ + next => createBucket(bucket, next), + next => + wait(WAIT_MS, () => { + checkMetrics(0, 0, 0); + next(); + }), + next => deleteBucket(bucket, next), + next => + wait(WAIT_MS, () => { + checkMetrics(0, 0, 0); + next(); + }), + ], + done, + ); }); it('should set metrics for putObject and deleteObject', done => { const bucket = 'bucket2'; @@ -226,75 +251,93 @@ describe('utapi v2 metrics incoming and outgoing bytes', function t() { const obj2Size = objectSize * 2; const key1 = '1.txt'; const key2 = '2.txt'; - async.series([ - next => createBucket(bucket, next), - next => putObject(bucket, key1, obj1Size, next), - next => wait(WAIT_MS, () => { - checkMetrics(obj1Size, 0, 1); - next(); - }), - next => putObject(bucket, key2, obj2Size, next), - next => wait(WAIT_MS, () => { - checkMetrics(obj1Size + obj2Size, 0, 2); - next(); - }), - next => deleteObject(bucket, key1, next), - next => wait(WAIT_MS, () => { - checkMetrics(obj2Size, 0, 1); - next(); - }), - next => deleteObject(bucket, key2, next), - next => wait(WAIT_MS, () => { - checkMetrics(0, 0, 0); - next(); - }), - next => deleteBucket(bucket, next), - ], done); + async.series( + [ + next => createBucket(bucket, next), + next => putObject(bucket, key1, obj1Size, next), + next => + wait(WAIT_MS, () => { + checkMetrics(obj1Size, 0, 1); + next(); + }), + next => putObject(bucket, key2, obj2Size, next), + next => + wait(WAIT_MS, () => { + checkMetrics(obj1Size + obj2Size, 0, 2); + next(); + }), + next => deleteObject(bucket, key1, next), + next => + wait(WAIT_MS, () => { + checkMetrics(obj2Size, 0, 1); + next(); + }), + next => deleteObject(bucket, key2, next), + next => + wait(WAIT_MS, () => { + checkMetrics(0, 0, 0); + next(); + }), + next => deleteBucket(bucket, next), + ], + done, + ); }); it('should set metrics for copyObject', done => { const bucket = 'bucket3'; const objectSize = 1024 * 1024 * 2; const key = '3.txt'; - async.series([ - next => createBucket(bucket, next), - next => putObject(bucket, key, objectSize, next), - next => copyObject(bucket, key, next), - next => wait(WAIT_MS, () => { - checkMetrics(objectSize * 2, 0, 2); - next(); - }), - next => deleteObject(bucket, key, next), - next => wait(WAIT_MS, () => { - checkMetrics(objectSize, 0, 1); - next(); - }), - next => deleteObject(bucket, `${key}-copy`, next), - next => wait(WAIT_MS, () => { - checkMetrics(0, 0, 0); - next(); - }), - next => deleteBucket(bucket, next), - ], done); + async.series( + [ + next => createBucket(bucket, next), + next => putObject(bucket, key, objectSize, next), + next => copyObject(bucket, key, next), + next => + wait(WAIT_MS, () => { + checkMetrics(objectSize * 2, 0, 2); + next(); + }), + next => deleteObject(bucket, key, next), + next => + wait(WAIT_MS, () => { + checkMetrics(objectSize, 0, 1); + next(); + }), + next => deleteObject(bucket, `${key}-copy`, next), + next => + wait(WAIT_MS, () => { + checkMetrics(0, 0, 0); + next(); + }), + next => deleteBucket(bucket, next), + ], + done, + ); }); it('should set metrics for getObject', done => { const bucket = 'bucket4'; const objectSize = 1024 * 1024 * 2; const key = '4.txt'; - async.series([ - next => createBucket(bucket, next), - next => putObject(bucket, key, objectSize, next), - next => getObject(bucket, key, next), - next => wait(WAIT_MS, () => { - checkMetrics(objectSize, objectSize, 1); - next(); - }), - next => deleteObject(bucket, key, next), - next => wait(WAIT_MS, () => { - checkMetrics(0, objectSize, 0); - next(); - }), - next => deleteBucket(bucket, next), - ], done); + async.series( + [ + next => createBucket(bucket, next), + next => putObject(bucket, key, objectSize, next), + next => getObject(bucket, key, next), + next => + wait(WAIT_MS, () => { + checkMetrics(objectSize, objectSize, 1); + next(); + }), + next => deleteObject(bucket, key, next), + next => + wait(WAIT_MS, () => { + checkMetrics(0, objectSize, 0); + next(); + }), + next => deleteBucket(bucket, next), + ], + done, + ); }); it('should set metrics for multiObjectDelete', done => { const bucket = 'bucket5'; @@ -303,133 +346,164 @@ describe('utapi v2 metrics incoming and outgoing bytes', function t() { const obj2Size = objectSize * 1; const key1 = '1.txt'; const key2 = '2.txt'; - async.series([ - next => createBucket(bucket, next), - next => putObject(bucket, key1, obj1Size, next), - next => wait(WAIT_MS, next), - next => putObject(bucket, key2, obj2Size, next), - next => wait(WAIT_MS, () => { - checkMetrics(obj1Size + obj2Size, 0, 2); - next(); - }), - next => deleteObjects(bucket, [key1, key2], next), - next => wait(WAIT_MS, () => { - checkMetrics(0, 0, 0); - next(); - }), - next => deleteBucket(bucket, next), - ], done); + async.series( + [ + next => createBucket(bucket, next), + next => putObject(bucket, key1, obj1Size, next), + next => wait(WAIT_MS, next), + next => putObject(bucket, key2, obj2Size, next), + next => + wait(WAIT_MS, () => { + checkMetrics(obj1Size + obj2Size, 0, 2); + next(); + }), + next => deleteObjects(bucket, [key1, key2], next), + next => + wait(WAIT_MS, () => { + checkMetrics(0, 0, 0); + next(); + }), + next => deleteBucket(bucket, next), + ], + done, + ); }); it('should set metrics for multiPartUpload', done => { const bucket = 'bucket6'; const partSize = 1024 * 1024 * 6; const parts = 2; const key = '6.txt'; - async.series([ - next => createBucket(bucket, next), - next => objectMPU(bucket, key, parts, partSize, next), - next => wait(WAIT_MS, () => { - checkMetrics(partSize * parts, 0, 1); - next(); - }), - next => deleteObject(bucket, key, next), - next => wait(WAIT_MS, () => { - checkMetrics(0, 0, 0); - next(); - }), - next => deleteBucket(bucket, next), - ], done); + async.series( + [ + next => createBucket(bucket, next), + next => objectMPU(bucket, key, parts, partSize, next), + next => + wait(WAIT_MS, () => { + checkMetrics(partSize * parts, 0, 1); + next(); + }), + next => deleteObject(bucket, key, next), + next => + wait(WAIT_MS, () => { + checkMetrics(0, 0, 0); + next(); + }), + next => deleteBucket(bucket, next), + ], + done, + ); }); it('should set metrics in versioned bucket', done => { const bucket = 'bucket7'; const objectSize = 1024 * 1024; const key = '7.txt'; - async.series([ - next => createBucket(bucket, next), - next => enableVersioning(bucket, true, next), - next => putObject(bucket, key, objectSize, next), - next => wait(WAIT_MS, next), - next => putObject(bucket, key, objectSize, next), - next => wait(WAIT_MS, () => { - checkMetrics(objectSize * 2, 0, 2); - next(); - }), - next => deleteObject(bucket, key, next), - next => wait(WAIT_MS, () => { - checkMetrics(objectSize * 2, 0, 3); - next(); - }), - next => removeVersions([bucket], next), - next => wait(WAIT_MS, () => { - checkMetrics(0, 0, 0); - next(); - }), - next => deleteBucket(bucket, next), - ], done); + async.series( + [ + next => createBucket(bucket, next), + next => enableVersioning(bucket, true, next), + next => putObject(bucket, key, objectSize, next), + next => wait(WAIT_MS, next), + next => putObject(bucket, key, objectSize, next), + next => + wait(WAIT_MS, () => { + checkMetrics(objectSize * 2, 0, 2); + next(); + }), + next => deleteObject(bucket, key, next), + next => + wait(WAIT_MS, () => { + checkMetrics(objectSize * 2, 0, 3); + next(); + }), + next => removeVersions([bucket], next), + next => + wait(WAIT_MS, () => { + checkMetrics(0, 0, 0); + next(); + }), + next => deleteBucket(bucket, next), + ], + done, + ); }); it('should set metrics for multipartUpload in a versioned bucket', done => { const bucket = 'bucket8'; const partSize = 1024 * 1024 * 6; const parts = 2; const key = '8.txt'; - async.series([ - next => createBucket(bucket, next), - next => enableVersioning(bucket, true, next), - next => objectMPU(bucket, key, parts, partSize, next), - next => wait(WAIT_MS, () => { - checkMetrics(partSize * parts, 0, 1); - next(); - }), - next => removeVersions([bucket], next), - next => wait(WAIT_MS, () => { - checkMetrics(0, 0, 0); - next(); - }), - next => deleteBucket(bucket, next), - ], done); + async.series( + [ + next => createBucket(bucket, next), + next => enableVersioning(bucket, true, next), + next => objectMPU(bucket, key, parts, partSize, next), + next => + wait(WAIT_MS, () => { + checkMetrics(partSize * parts, 0, 1); + next(); + }), + next => removeVersions([bucket], next), + next => + wait(WAIT_MS, () => { + checkMetrics(0, 0, 0); + next(); + }), + next => deleteBucket(bucket, next), + ], + done, + ); }); it('should set metrics for multipartUpload overwrite in a versioned bucket', done => { const bucket = 'bucket9'; const partSize = 1024 * 1024 * 6; const parts = 2; const key = '9.txt'; - async.series([ - next => createBucket(bucket, next), - next => enableVersioning(bucket, true, next), - next => objectMPU(bucket, key, parts, partSize, next), - next => objectMPU(bucket, key, parts, partSize, next), - next => wait(WAIT_MS, () => { - checkMetrics(partSize * parts * 2, 0, 2); - next(); - }), - next => removeVersions([bucket], next), - next => wait(WAIT_MS, () => { - checkMetrics(0, 0, 0); - next(); - }), - next => deleteBucket(bucket, next), - ], done); + async.series( + [ + next => createBucket(bucket, next), + next => enableVersioning(bucket, true, next), + next => objectMPU(bucket, key, parts, partSize, next), + next => objectMPU(bucket, key, parts, partSize, next), + next => + wait(WAIT_MS, () => { + checkMetrics(partSize * parts * 2, 0, 2); + next(); + }), + next => removeVersions([bucket], next), + next => + wait(WAIT_MS, () => { + checkMetrics(0, 0, 0); + next(); + }), + next => deleteBucket(bucket, next), + ], + done, + ); }); it('should set metrics for multiPartUpload overwrite', done => { const bucket = 'bucket10'; const partSize = 1024 * 1024 * 6; const parts = 2; const key = '10.txt'; - async.series([ - next => createBucket(bucket, next), - next => objectMPU(bucket, key, parts, partSize, next), - next => objectMPU(bucket, key, parts, partSize, next), - next => wait(WAIT_MS, () => { - checkMetrics(partSize * parts, 0, 1); - next(); - }), - next => deleteObject(bucket, key, next), - next => wait(WAIT_MS, () => { - checkMetrics(0, 0, 0); - next(); - }), - next => deleteBucket(bucket, next), - ], done); + async.series( + [ + next => createBucket(bucket, next), + next => objectMPU(bucket, key, parts, partSize, next), + next => objectMPU(bucket, key, parts, partSize, next), + next => + wait(WAIT_MS, () => { + checkMetrics(partSize * parts, 0, 1); + next(); + }), + next => deleteObject(bucket, key, next), + next => + wait(WAIT_MS, () => { + checkMetrics(0, 0, 0); + next(); + }), + next => deleteBucket(bucket, next), + ], + done, + ); }); it('should set metrics for multiObjectDelete in a versioned bucket', done => { const bucket = 'bucket11'; @@ -438,47 +512,58 @@ describe('utapi v2 metrics incoming and outgoing bytes', function t() { const obj2Size = objectSize * 1; const key1 = '1.txt'; const key2 = '2.txt'; - async.series([ - next => createBucket(bucket, next), - next => enableVersioning(bucket, true, next), - next => putObject(bucket, key1, obj1Size, next), - next => wait(WAIT_MS, next), - next => putObject(bucket, key2, obj2Size, next), - next => wait(WAIT_MS, () => { - checkMetrics(obj1Size + obj2Size, 0, 2); - next(); - }), - next => deleteObjects(bucket, [key1, key2], next), - next => wait(WAIT_MS, () => { - checkMetrics(obj1Size + obj2Size, 0, 4); - next(); - }), - next => removeVersions([bucket], next), - next => wait(WAIT_MS, () => { - checkMetrics(0, 0, 0); - next(); - }), - next => deleteBucket(bucket, next), - ], done); + async.series( + [ + next => createBucket(bucket, next), + next => enableVersioning(bucket, true, next), + next => putObject(bucket, key1, obj1Size, next), + next => wait(WAIT_MS, next), + next => putObject(bucket, key2, obj2Size, next), + next => + wait(WAIT_MS, () => { + checkMetrics(obj1Size + obj2Size, 0, 2); + next(); + }), + next => deleteObjects(bucket, [key1, key2], next), + next => + wait(WAIT_MS, () => { + checkMetrics(obj1Size + obj2Size, 0, 4); + next(); + }), + next => removeVersions([bucket], next), + next => + wait(WAIT_MS, () => { + checkMetrics(0, 0, 0); + next(); + }), + next => deleteBucket(bucket, next), + ], + done, + ); }); it('should not push a metric for a filtered bucket', done => { const bucket = 'utapi-event-filter-deny-bucket'; const objSize = 2 * 1024 * 1024; const key = '1.txt'; - async.series([ - next => createBucket(bucket, next), - next => putObject(bucket, key, objSize, next), - next => wait(WAIT_MS, () => { - checkMetrics(0, 0, 0); - next(); - }), - next => deleteObject(bucket, key, next), - next => wait(WAIT_MS, () => { - checkMetrics(0, 0, 0); - next(); - }), - next => deleteBucket(bucket, next), - ], done); + async.series( + [ + next => createBucket(bucket, next), + next => putObject(bucket, key, objSize, next), + next => + wait(WAIT_MS, () => { + checkMetrics(0, 0, 0); + next(); + }), + next => deleteObject(bucket, key, next), + next => + wait(WAIT_MS, () => { + checkMetrics(0, 0, 0); + next(); + }), + next => deleteBucket(bucket, next), + ], + done, + ); }); }); diff --git a/tests/utapi/utilities.js b/tests/utapi/utilities.js index b8b7295f9c..5046e029b9 100644 --- a/tests/utapi/utilities.js +++ b/tests/utapi/utilities.js @@ -4,481 +4,520 @@ const werelogs = require('werelogs'); const _config = require('../../lib/Config').config; const { makeAuthInfo } = require('../unit/helpers'); -const testEvents = [{ - action: 'getObject', - metrics: { - bucket: 'bucket1', - keys: ['1.txt'], - newByteLength: 2, - oldByteLength: null, - versionId: undefined, - location: 'us-west-1', - numberOfObjects: 1, - byteLength: null, - isDelete: false, - }, - expected: { - objectDelta: 1, - sizeDelta: 0, - incomingBytes: 0, - outgoingBytes: 2, - }, -}, { - action: 'deleteObject', - metrics: { - bucket: 'bucket1', - keys: ['1.txt'], - byteLength: 2, - numberOfObjects: 1, - location: 'us-west-1', - isDelete: true, - }, - expected: { - objectDelta: -1, - sizeDelta: -2, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'listBucket', - metrics: { - bucket: 'bucket1', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'putObject', - metrics: { - bucket: 'bucket1', - keys: ['2.txt'], - newByteLength: 2, - oldByteLength: null, - versionId: undefined, - location: 'us-west-1', - numberOfObjects: 1, - }, - expected: { - objectDelta: 1, - sizeDelta: 2, - incomingBytes: 2, - outgoingBytes: 0, - }, -}, { - action: 'listBucket', - metrics: { - bucket: 'bucket1', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'headObject', - metrics: { - bucket: 'bucket1', - keys: ['1.txt'], - versionId: undefined, - location: 'us-west-1', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'abortMultipartUpload', - metrics: { - bucket: 'destinationbucket815502017', - keys: ['copycatobject'], - byteLength: 26, - location: 'us-east-1', - }, - expected: { - objectDelta: undefined, - sizeDelta: -26, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'completeMultipartUpload', - metrics: { - oldByteLength: null, - bucket: 'destinationbucket815502017', - keys: ['copycatobject'], - versionId: undefined, - numberOfObjects: 1, - location: 'us-east-1', - }, - expected: { - objectDelta: 1, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'createBucket', - metrics: { - bucket: 'deletebucketpolicy-test-bucket', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'deleteBucket', - metrics: { - bucket: 'deletebucketpolicy-test-bucket', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'deleteBucketCors', - metrics: { - bucket: 'testdeletecorsbucket', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'deleteBucketReplication', - metrics: { - bucket: 'source-bucket', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'deleteBucketWebsite', - metrics: { - bucket: 'testdeletewebsitebucket', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'getBucketAcl', - metrics: { - bucket: 'putbucketaclfttest', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'getBucketCors', - metrics: { - bucket: 'testgetcorsbucket', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'getBucketLocation', - metrics: { - bucket: 'testgetlocationbucket', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'getBucketNotification', - metrics: { - bucket: 'notificationtestbucket', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'getBucketObjectLock', - metrics: { - bucket: 'mock-bucket', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'getBucketReplication', - metrics: { - bucket: 'source-bucket', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'getBucketVersioning', - metrics: { - bucket: 'bucket-with-object-lock', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'getBucketWebsite', - metrics: { - bucket: 'testgetwebsitetestbucket', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'getObjectTagging', - metrics: { - bucket: 'completempu1615102906771', - keys: ['keywithtags'], - versionId: undefined, - location: 'us-east-1', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'headObject', - metrics: { - bucket: 'supersourcebucket81033016532', - keys: ['supersourceobject'], - versionId: undefined, - location: 'us-east-1', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'initiateMultipartUpload', - metrics: { - bucket: 'destinationbucket815502017', - keys: ['copycatobject'], - location: 'us-east-1', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'listMultipartUploadParts', - metrics: { - bucket: 'ftest-mybucket-74', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'multiObjectDelete', - metrics: { - bucket: 'completempu1615102906771', - keys: [undefined], - byteLength: 3, - numberOfObjects: 1, - removedDeleteMarkers: 1, - isDelete: true, - }, - expected: { - objectDelta: -2, - sizeDelta: -3, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'putBucketAcl', - metrics: { - bucket: 'putbucketaclfttest', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'putBucketCors', - metrics: { - bucket: 'testcorsbucket', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'putBucketNotification', - metrics: { - bucket: 'notificationtestbucket', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'putBucketObjectLock', - metrics: { - bucket: 'mock-bucket', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'putBucketReplication', - metrics: { - bucket: 'source-bucket', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'putBucketVersioning', - metrics: { - bucket: 'source-bucket', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'putBucketWebsite', - metrics: { - bucket: 'testgetwebsitetestbucket', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'uploadPart', - metrics: { - bucket: 'ftest-mybucket-74', - keys: ['toAbort&<>"\''], - newByteLength: 5242880, - oldByteLength: null, - location: 'us-east-1', - }, - expected: { - objectDelta: undefined, - sizeDelta: 5242880, - incomingBytes: 5242880, - outgoingBytes: 0, - }, -}, { - action: 'uploadPartCopy', - metrics: { - bucket: 'destinationbucket815502017', - keys: ['copycatobject'], - newByteLength: 26, - oldByteLength: null, - location: 'us-east-1', - }, - expected: { - objectDelta: undefined, - sizeDelta: 26, - incomingBytes: 26, - outgoingBytes: 0, - }, -}, { - action: 'replicateObject', - metrics: { - bucket: 'source-bucket', - keys: ['mykey'], - newByteLength: 26, - oldByteLength: null, - }, - expected: { - objectDelta: 1, - sizeDelta: 26, - incomingBytes: 26, - outgoingBytes: 0, - }, -}, { - action: 'replicateDelete', - metrics: { - bucket: 'source-bucket', - keys: ['mykey'], - }, - expected: { - objectDelta: 1, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'replicateTags', - metrics: { - bucket: 'source-bucket', - keys: ['mykey'], - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}]; +const testEvents = [ + { + action: 'getObject', + metrics: { + bucket: 'bucket1', + keys: ['1.txt'], + newByteLength: 2, + oldByteLength: null, + versionId: undefined, + location: 'us-west-1', + numberOfObjects: 1, + byteLength: null, + isDelete: false, + }, + expected: { + objectDelta: 1, + sizeDelta: 0, + incomingBytes: 0, + outgoingBytes: 2, + }, + }, + { + action: 'deleteObject', + metrics: { + bucket: 'bucket1', + keys: ['1.txt'], + byteLength: 2, + numberOfObjects: 1, + location: 'us-west-1', + isDelete: true, + }, + expected: { + objectDelta: -1, + sizeDelta: -2, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'listBucket', + metrics: { + bucket: 'bucket1', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'putObject', + metrics: { + bucket: 'bucket1', + keys: ['2.txt'], + newByteLength: 2, + oldByteLength: null, + versionId: undefined, + location: 'us-west-1', + numberOfObjects: 1, + }, + expected: { + objectDelta: 1, + sizeDelta: 2, + incomingBytes: 2, + outgoingBytes: 0, + }, + }, + { + action: 'listBucket', + metrics: { + bucket: 'bucket1', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'headObject', + metrics: { + bucket: 'bucket1', + keys: ['1.txt'], + versionId: undefined, + location: 'us-west-1', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'abortMultipartUpload', + metrics: { + bucket: 'destinationbucket815502017', + keys: ['copycatobject'], + byteLength: 26, + location: 'us-east-1', + }, + expected: { + objectDelta: undefined, + sizeDelta: -26, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'completeMultipartUpload', + metrics: { + oldByteLength: null, + bucket: 'destinationbucket815502017', + keys: ['copycatobject'], + versionId: undefined, + numberOfObjects: 1, + location: 'us-east-1', + }, + expected: { + objectDelta: 1, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'createBucket', + metrics: { + bucket: 'deletebucketpolicy-test-bucket', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'deleteBucket', + metrics: { + bucket: 'deletebucketpolicy-test-bucket', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'deleteBucketCors', + metrics: { + bucket: 'testdeletecorsbucket', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'deleteBucketReplication', + metrics: { + bucket: 'source-bucket', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'deleteBucketWebsite', + metrics: { + bucket: 'testdeletewebsitebucket', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'getBucketAcl', + metrics: { + bucket: 'putbucketaclfttest', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'getBucketCors', + metrics: { + bucket: 'testgetcorsbucket', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'getBucketLocation', + metrics: { + bucket: 'testgetlocationbucket', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'getBucketNotification', + metrics: { + bucket: 'notificationtestbucket', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'getBucketObjectLock', + metrics: { + bucket: 'mock-bucket', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'getBucketReplication', + metrics: { + bucket: 'source-bucket', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'getBucketVersioning', + metrics: { + bucket: 'bucket-with-object-lock', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'getBucketWebsite', + metrics: { + bucket: 'testgetwebsitetestbucket', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'getObjectTagging', + metrics: { + bucket: 'completempu1615102906771', + keys: ['keywithtags'], + versionId: undefined, + location: 'us-east-1', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'headObject', + metrics: { + bucket: 'supersourcebucket81033016532', + keys: ['supersourceobject'], + versionId: undefined, + location: 'us-east-1', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'initiateMultipartUpload', + metrics: { + bucket: 'destinationbucket815502017', + keys: ['copycatobject'], + location: 'us-east-1', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'listMultipartUploadParts', + metrics: { + bucket: 'ftest-mybucket-74', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'multiObjectDelete', + metrics: { + bucket: 'completempu1615102906771', + keys: [undefined], + byteLength: 3, + numberOfObjects: 1, + removedDeleteMarkers: 1, + isDelete: true, + }, + expected: { + objectDelta: -2, + sizeDelta: -3, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'putBucketAcl', + metrics: { + bucket: 'putbucketaclfttest', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'putBucketCors', + metrics: { + bucket: 'testcorsbucket', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'putBucketNotification', + metrics: { + bucket: 'notificationtestbucket', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'putBucketObjectLock', + metrics: { + bucket: 'mock-bucket', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'putBucketReplication', + metrics: { + bucket: 'source-bucket', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'putBucketVersioning', + metrics: { + bucket: 'source-bucket', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'putBucketWebsite', + metrics: { + bucket: 'testgetwebsitetestbucket', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'uploadPart', + metrics: { + bucket: 'ftest-mybucket-74', + keys: ['toAbort&<>"\''], + newByteLength: 5242880, + oldByteLength: null, + location: 'us-east-1', + }, + expected: { + objectDelta: undefined, + sizeDelta: 5242880, + incomingBytes: 5242880, + outgoingBytes: 0, + }, + }, + { + action: 'uploadPartCopy', + metrics: { + bucket: 'destinationbucket815502017', + keys: ['copycatobject'], + newByteLength: 26, + oldByteLength: null, + location: 'us-east-1', + }, + expected: { + objectDelta: undefined, + sizeDelta: 26, + incomingBytes: 26, + outgoingBytes: 0, + }, + }, + { + action: 'replicateObject', + metrics: { + bucket: 'source-bucket', + keys: ['mykey'], + newByteLength: 26, + oldByteLength: null, + }, + expected: { + objectDelta: 1, + sizeDelta: 26, + incomingBytes: 26, + outgoingBytes: 0, + }, + }, + { + action: 'replicateDelete', + metrics: { + bucket: 'source-bucket', + keys: ['mykey'], + }, + expected: { + objectDelta: 1, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'replicateTags', + metrics: { + bucket: 'source-bucket', + keys: ['mykey'], + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, +]; describe('utapi v2 pushmetrics utility', () => { const log = new werelogs.Logger('utapi-utility'); @@ -491,8 +530,7 @@ describe('utapi v2 pushmetrics utility', () => { before(() => { assert.strictEqual(utapiVersion, 2); - sinon.stub(UtapiClient.prototype, 'pushMetric') - .callsFake(pushMetricStub); + sinon.stub(UtapiClient.prototype, 'pushMetric').callsFake(pushMetricStub); pushMetric = require('../../lib/utapi/utilities').pushMetric; }); @@ -530,34 +568,34 @@ describe('utapi v2 pushmetrics utility', () => { ]); testEvents - .map(event => { - const modifiedEvent = event; - const authInfo = makeAuthInfo('accesskey1', 'Bart'); - authInfo.arn = `foo:assumed-role/${_config.lifecycleRoleName}/backbeat-lifecycle`; - modifiedEvent.metrics.authInfo = authInfo; - modifiedEvent.metrics.canonicalID = 'accesskey1'; - return modifiedEvent; - }) - .map(event => { - if (eventFilterList.has(event.action)) { - it(`should skip action ${event.action}`, () => { - _config.lifecycleRoleName = 'lifecycleTestRoleName'; - const eventPushed = pushMetric(event.action, log, event.metrics); - assert.strictEqual(eventPushed, undefined); - }); - } - return event; - }) - .forEach(event => { - if (!eventFilterList.has(event.action)) { - it(`should compute and push metrics for ${event.action}`, () => { - const eventPushed = pushMetric(event.action, log, event.metrics); - assert(eventPushed); - Object.keys(event.expected).forEach(key => { - assert.strictEqual(eventPushed[key], event.expected[key]); + .map(event => { + const modifiedEvent = event; + const authInfo = makeAuthInfo('accesskey1', 'Bart'); + authInfo.arn = `foo:assumed-role/${_config.lifecycleRoleName}/backbeat-lifecycle`; + modifiedEvent.metrics.authInfo = authInfo; + modifiedEvent.metrics.canonicalID = 'accesskey1'; + return modifiedEvent; + }) + .map(event => { + if (eventFilterList.has(event.action)) { + it(`should skip action ${event.action}`, () => { + _config.lifecycleRoleName = 'lifecycleTestRoleName'; + const eventPushed = pushMetric(event.action, log, event.metrics); + assert.strictEqual(eventPushed, undefined); }); - }); - } - }); + } + return event; + }) + .forEach(event => { + if (!eventFilterList.has(event.action)) { + it(`should compute and push metrics for ${event.action}`, () => { + const eventPushed = pushMetric(event.action, log, event.metrics); + assert(eventPushed); + Object.keys(event.expected).forEach(key => { + assert.strictEqual(eventPushed[key], event.expected[key]); + }); + }); + } + }); }); }); diff --git a/tests/utilities/bucketTagging-util.js b/tests/utilities/bucketTagging-util.js index 337780c27a..aca1d02b28 100644 --- a/tests/utilities/bucketTagging-util.js +++ b/tests/utilities/bucketTagging-util.js @@ -5,11 +5,17 @@ function assertError(err, expectedErr) { if (expectedErr === null) { assert.strictEqual(err, null, `expected no error but got '${err}'`); } else { - assert.strictEqual(err.Code, expectedErr, 'incorrect error response ' + - `code: should be '${expectedErr}' but got '${err.Code}'`); - assert.strictEqual(err.$metadata.httpStatusCode, errors[expectedErr].code, + assert.strictEqual( + err.Code, + expectedErr, + 'incorrect error response ' + `code: should be '${expectedErr}' but got '${err.Code}'`, + ); + assert.strictEqual( + err.$metadata.httpStatusCode, + errors[expectedErr].code, 'incorrect error status code: should be ' + - `${errors[expectedErr].code}, but got '${err.$metadata.httpStatusCode}'`); + `${errors[expectedErr].code}, but got '${err.$metadata.httpStatusCode}'`, + ); } } diff --git a/tests/utilities/mock/Scuba.js b/tests/utilities/mock/Scuba.js index 7dbfa49c72..a2996029ff 100644 --- a/tests/utilities/mock/Scuba.js +++ b/tests/utilities/mock/Scuba.js @@ -42,7 +42,8 @@ class Scuba { }); const immediateInflights = req.body?.action === 'objectRestore' ? 0 : inflight; return res.json({ - bytesTotal: (this._data.bucket.get(bucketName)?.current || 0) + + bytesTotal: + (this._data.bucket.get(bucketName)?.current || 0) + (this._data.bucket.get(bucketName)?.nonCurrent || 0) + (this._data.bucket.get(bucketName)?.inflight || 0) + immediateInflights, @@ -116,7 +117,7 @@ class Scuba { let inflightCount = 0; this._data.bucket.forEach((value, key) => { if (!this.supportsInflight && key === bucketName) { - inflightCount += (value.current + value.nonCurrent); + inflightCount += value.current + value.nonCurrent; } else if (this.supportsInflight && key.startsWith(`${bucketName}_`)) { inflightCount += value.inflight; } diff --git a/tests/utilities/objectLock-util.js b/tests/utilities/objectLock-util.js index 92a223e6cd..e9cae0212c 100644 --- a/tests/utilities/objectLock-util.js +++ b/tests/utilities/objectLock-util.js @@ -10,22 +10,26 @@ const versionIdUtils = versioning.VersionID; const log = new DummyRequestLogger(); function changeObjectLock(objects, newConfig, cb) { - async.each(objects, (object, next) => { - const { bucket, key, versionId } = object; - metadataGetObject(bucket, key, versionIdUtils.decode(versionId), null, log, (err, objMD) => { - assert.ifError(err); - // set newConfig as empty string to remove object lock - /* eslint-disable no-param-reassign */ - objMD.retentionMode = newConfig.mode; - objMD.retentionDate = newConfig.date; - objMD.legalHold = false; - const params = { versionId: objMD.versionId, isNull: false }; - metadata.putObjectMD(bucket, key, objMD, params, log, err => { + async.each( + objects, + (object, next) => { + const { bucket, key, versionId } = object; + metadataGetObject(bucket, key, versionIdUtils.decode(versionId), null, log, (err, objMD) => { assert.ifError(err); - next(); + // set newConfig as empty string to remove object lock + /* eslint-disable no-param-reassign */ + objMD.retentionMode = newConfig.mode; + objMD.retentionDate = newConfig.date; + objMD.legalHold = false; + const params = { versionId: objMD.versionId, isNull: false }; + metadata.putObjectMD(bucket, key, objMD, params, log, err => { + assert.ifError(err); + next(); + }); }); - }); - }, cb); + }, + cb, + ); } module.exports = changeObjectLock; diff --git a/yamllint.yml b/yamllint.yml index 1a62f98541..a47c310d44 100644 --- a/yamllint.yml +++ b/yamllint.yml @@ -1,5 +1,4 @@ --- - extends: default rules: From 78f78503b0db056b3205e4a17e202e97cc82b7f8 Mon Sep 17 00:00:00 2001 From: DarkIsDude Date: Mon, 21 Sep 2026 15:03:11 +0200 Subject: [PATCH 3/6] =?UTF-8?q?=F0=9F=92=9A=20keep=20the=20reformatted=20t?= =?UTF-8?q?able=20within=20mdlint's=20line=20limit?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prettier pads markdown table cells for alignment, which pushed this table to 82 columns and tripped MD013. mdlint's config lives in the shared Guidelines package and cannot be relaxed per repo, so shorten the widest cell instead. Issue: CLDSRV-1002 --- CLAUDE.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 53cbb1fdaa..8c2eca5bfa 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -108,12 +108,12 @@ backends defined in `locationConfig.json` based on location constraints. ### Metadata Backends (`S3METADATA`) -| Backend | Port | Description | -| ---------------- | ------ | -------------------------------------------------- | -| `file` (default) | 9990 | Local LevelDB via `mdserver.js` | -| `scality` | 9000 | External bucketd service (production Scality RING) | -| `mongodb` | 27017+ | MongoDB replica set | -| `mem` | - | In-memory (testing only) | +| Backend | Port | Description | +| ---------------- | ------ | ------------------------------------------ | +| `file` (default) | 9990 | Local LevelDB via `mdserver.js` | +| `scality` | 9000 | External bucketd (production Scality RING) | +| `mongodb` | 27017+ | MongoDB replica set | +| `mem` | - | In-memory (testing only) | **file vs scality**: The `file` backend runs a self-contained metadata server (`mdserver.js`) for development. The `scality` backend connects to external From 2f66c2b527d8380d09a77a561396b64c5cd82a4c Mon Sep 17 00:00:00 2001 From: DarkIsDude Date: Mon, 21 Sep 2026 15:06:48 +0200 Subject: [PATCH 4/6] =?UTF-8?q?=F0=9F=92=9A=20keep=20the=20website=20test?= =?UTF-8?q?=20fixtures=20unformatted?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit websiteHead.js and websiteHeadWithACL.js assert the ETag of index.html, so reformatting these fixtures changes their MD5 and breaks the tests. Their bytes are the test data, not source to style. Issue: CLDSRV-1002 --- .prettierignore | 3 +++ .../test/object/websiteFiles/error.html | 14 +++++++------- .../test/object/websiteFiles/index.html | 17 ++++++++--------- .../test/object/websiteFiles/redirect.html | 12 ++++++------ 4 files changed, 24 insertions(+), 22 deletions(-) diff --git a/.prettierignore b/.prettierignore index 49284af952..4eadc69b4d 100644 --- a/.prettierignore +++ b/.prettierignore @@ -2,3 +2,6 @@ coverage/ localData/ localMetadata/ junit/ + +# Website fixtures: their exact bytes are asserted as ETags in tests +tests/functional/aws-node-sdk/test/object/websiteFiles/ diff --git a/tests/functional/aws-node-sdk/test/object/websiteFiles/error.html b/tests/functional/aws-node-sdk/test/object/websiteFiles/error.html index 5c73b35224..71f5a25838 100644 --- a/tests/functional/aws-node-sdk/test/object/websiteFiles/error.html +++ b/tests/functional/aws-node-sdk/test/object/websiteFiles/error.html @@ -1,9 +1,9 @@ - - Error!! - - -

It appears you messed up

-

Or maybe it was me...

- + + Error!! + + +

It appears you messed up

+

Or maybe it was me...

+ diff --git a/tests/functional/aws-node-sdk/test/object/websiteFiles/index.html b/tests/functional/aws-node-sdk/test/object/websiteFiles/index.html index 832b3d787a..8ce654d9a3 100644 --- a/tests/functional/aws-node-sdk/test/object/websiteFiles/index.html +++ b/tests/functional/aws-node-sdk/test/object/websiteFiles/index.html @@ -1,11 +1,10 @@ - - Best testing website ever - - -

Welcome to my extraordinary bucket website testing page

-

Now hosted on Scality's S3 Server -- a symphonic storage experience!

-
- - + + Best testing website ever + + +

Welcome to my extraordinary bucket website testing page

+

Now hosted on Scality's S3 Server -- a symphonic storage experience!

+
+ diff --git a/tests/functional/aws-node-sdk/test/object/websiteFiles/redirect.html b/tests/functional/aws-node-sdk/test/object/websiteFiles/redirect.html index b16f62eebd..1f02b665d9 100644 --- a/tests/functional/aws-node-sdk/test/object/websiteFiles/redirect.html +++ b/tests/functional/aws-node-sdk/test/object/websiteFiles/redirect.html @@ -1,8 +1,8 @@ - - Best redirect link ever - - -

Welcome to your redirection file

- + + Best redirect link ever + + +

Welcome to your redirection file

+ From e17392f05aeccb6278d49a0d2c27e8db22c9e542 Mon Sep 17 00:00:00 2001 From: DarkIsDude Date: Mon, 21 Sep 2026 15:42:31 +0200 Subject: [PATCH 5/6] =?UTF-8?q?=F0=9F=92=9A=20drop=20the=20max-len=20disab?= =?UTF-8?q?le=20directives=20left=20on=209.4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These files do not exist on 9.3, so the cleanup that came with the prettier switch never reached them. With max-len off they are unused directives, and the strict lint step runs with --max-warnings 0. Issue: CLDSRV-1002 --- .../functional/raw-node/test/checksumPutObjectUploadPart.js | 1 - tests/sur/quota.js | 1 - tests/unit/api/apiUtils/object/storeObject.js | 5 ----- 3 files changed, 7 deletions(-) diff --git a/tests/functional/raw-node/test/checksumPutObjectUploadPart.js b/tests/functional/raw-node/test/checksumPutObjectUploadPart.js index 165d9406f1..c4502f41c2 100644 --- a/tests/functional/raw-node/test/checksumPutObjectUploadPart.js +++ b/tests/functional/raw-node/test/checksumPutObjectUploadPart.js @@ -637,7 +637,6 @@ function makeScenarioTests(urlFn, { expectsImplicitChecksum = true } = {}) { itSkipIfAWS('should return 200 for trailer line with whitespace around name and value', done => { // TrailingChecksumTransform trims both name and value, so whitespace is accepted. - // eslint-disable-next-line max-len -- prettier keeps this fixture template on one line (121 > 120) const body = `f\r\ntrailer content\r\n0\r\n x-amz-checksum-sha256 : ${trailerContentSha256} \n\r\n\r\n\r\n`; doPutRequest( urlFn(), diff --git a/tests/sur/quota.js b/tests/sur/quota.js index 95e84372eb..5c22c03572 100644 --- a/tests/sur/quota.js +++ b/tests/sur/quota.js @@ -1159,7 +1159,6 @@ function multiObjectDelete(bucket, keys, size, callback) { ); }); - // eslint-disable-next-line max-len it('should only evaluate quota and not update inflights for PutObject with the x-scal-s3-version-id header', done => { const bucket = 'quota-test-bucket13'; const key = 'quota-test-object'; diff --git a/tests/unit/api/apiUtils/object/storeObject.js b/tests/unit/api/apiUtils/object/storeObject.js index 9270719c0b..fc5bfb0e2e 100644 --- a/tests/unit/api/apiUtils/object/storeObject.js +++ b/tests/unit/api/apiUtils/object/storeObject.js @@ -206,7 +206,6 @@ describe('dataStore', () => { }); }); - // eslint-disable-next-line max-len it('should wait for finish before validating when checksumedStream is not yet writableFinished after data.put', done => { let capturedStream; putStub.callsFake((cipher, stream, size, ctx, backend, log2, cb) => { @@ -233,7 +232,6 @@ describe('dataStore', () => { }); }); - // eslint-disable-next-line max-len it('should delete stored data and call cb with the error when checksumedStream emits error after data.put', done => { batchDeleteSucceeds(); let capturedStream; @@ -364,7 +362,6 @@ describe('dataStore', () => { }); describe('x-amz-content-sha256 body validation', () => { - // eslint-disable-next-line max-len it('should call cb with XAmzContentSHA256Mismatch and delete stored data when the hash does not match', done => { batchDeleteSucceeds(); putSucceeds(); @@ -406,7 +403,6 @@ describe('dataStore', () => { }); }); - // eslint-disable-next-line max-len it('should call cb with XAmzContentSHA256Mismatch when the hash mismatches and batchDelete also fails', done => { batchDeleteStub.callsFake((keys, a, b, log2, cb) => cb(errors.InternalError)); putSucceeds(); @@ -516,7 +512,6 @@ describe('dataStore', () => { }); }); - // eslint-disable-next-line max-len it('should call cb with stream error when checksumedStream errors after data.put and batchDelete also fails', done => { batchDeleteStub.callsFake((keys, a, b, log2, cb) => cb(errors.BadRequest)); let capturedStream; From 87241667266981ad152ab2552d6fb06be1946ddf Mon Sep 17 00:00:00 2001 From: DarkIsDude Date: Mon, 21 Sep 2026 15:55:28 +0200 Subject: [PATCH 6/6] =?UTF-8?q?=F0=9F=93=9D=20ignore=20the=20reformat=20co?= =?UTF-8?q?mmit=20in=20git=20blame?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GitHub reads .git-blame-ignore-revs automatically; locally it needs git config blame.ignoreRevsFile .git-blame-ignore-revs Issue: CLDSRV-1002 --- .git-blame-ignore-revs | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .git-blame-ignore-revs diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 0000000000..355fb965ca --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,3 @@ +# Whole-repo Prettier reformat (CLDSRV-1002), no behaviour change. +# Run once locally: git config blame.ignoreRevsFile .git-blame-ignore-revs +02f94246fe9d3f9e4484d526adc81a73e90a0540